diff --git a/.gitattributes b/.gitattributes index ec589a92ef97bd0f59b84cf320394ed5ad60a3fe..62abbfcddcae2e28cb4b119653c1709e5d474a9f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -922,3 +922,157 @@ FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6/OmniAv FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6/wandb/run-20260423_231648-1djswvuo/files/media/videos/val0/reconstructed_800_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6/wandb/run-20260423_231648-1djswvuo/files/media/videos/val0/reconstructed_800_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6/wandb/run-20260423_231648-1djswvuo/run-1djswvuo.wandb filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/media/videos/train_media/student/generation_100_04a9b61f163a19decce9.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/run-lxlsugeu.wandb filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_100_cefe94f8f6eedfc62861.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_200_522fcaa3ca88be5f5888.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_300_808c915bfaced95fd333.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_400_34bbda270541d22aac7e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_500_67ac3650ef016bc0d866.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_0d36a47ac23df8c49717.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_0f8b9fa5e2c061f798da.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_2cd6ea1c10d36df608f4.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_33508d91bb16115c4d95.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_764975d52a365b7007c1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_777d1b756b1466b170dd.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_78a815adf034840ead32.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_abc3526eae4ce7b95074.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_d1eda7c8bdee077184bd.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_ecf3b3bd0d2616edcd0d.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_3ddf326081c0796f3a13.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_434f141d2e23c05bd74d.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_4a28a1edd2355b9b361f.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_4c53a95f289f22e345ce.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_54b5ffc7e997ed3e03fe.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_683f35385b67a977e4ce.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_7aef4ffdb8a10e17e5e9.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_857e12191ec0dfaedc56.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_c97cf0d22a4f3855bf97.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_ed471b69e01df1aa7cd9.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_063074a6e3d9153b241b.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_0cbdd8e709e49641cc46.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_1fb55a14d14f5a0848cd.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_7ed7e3c953e4f537d576.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_9b077cec2e4d4dc35c9a.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_ad6468cffc81cdef7dfc.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_b0ea71e01862a49cf33f.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_bfeee8b5be654677567c.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_d13343fbf8c221ef568f.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_fb20fb759d5b7e6408fd.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_27e288e1f9c7c94078bd.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_2e45c60112dbc1dfddbb.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_3cba6a3368b82ac19add.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_5d2f0c3ef8f4f3d37d4a.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_91bc2d478d9b4731fdc5.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_942baaa2ff6d38d3e1de.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_a7fea23d34df802b17c1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_ba74a72801db8ecf90b1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_eae35008e76a476df277.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_f37b8bd7ac458e989f64.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_08b64e302ec053e09f7a.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_364210d1d39487ed18dd.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_382dcc9d28af35e517dd.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_415f82bb3b3e9cd393a0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_52ceb51262f49782f4b9.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_90b0c03047027618dec6.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_b22bbc588eb2412872cb.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_b6e3fb89f026804ddf55.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_e2e0efb6d3340c8a74b5.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_f565d15af15a1ba1d0af.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/run-3fi9xe8g.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..fd122c4fd87ff338d59fa383f8a97f0bc1d32c7a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6deabb3380d56a0f59d39b5ca8eec399e11289b758a70f64ba6eee0c4b55828 +size 614460 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..60155bde7c4e3f1b8905c686b78a5fa3843d8124 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e80c27c5697d5855d3263ab8ce77c0ba8cd655a8e4c7ba8b8ff076bcc36d7c10 +size 1424446005 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a88cf783f0657ea843676d9da4cb33c720382f30 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26d144a1f64d092c010c88d58fce2a35515ae98466665fd736c354d4b8ff5b15 +size 1401881227 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f1c17c284ae00f4f6a8dd1dd50cb1b42e7c37298 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c389fb388ad8dec4a9592421ed3a5091cf436ac61ea52b64a14be4c3d6afb7d5 +size 1406759189 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..373e664a2de089d375cc58fa73fee516236c08e7 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55c7d2ec627f6bbb6ce4b196265bd4080dbcdb385056203de5c94812ee739a00 +size 1401210558 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.pth b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.pth new file mode 100644 index 0000000000000000000000000000000000000000..8eba06d5534a7b2e93f2f5f3f62b7a9807edd0ae --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fefe455129d8d9683d7f02cdbb18043a7c8a6b6082c2b52130407de1795c467e +size 1901 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..38d13c779c5dcf4d92336bc44b04db4f6c20a977 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6f72a9e5ad4eba9299e0fcc91da8c2429270ffbb5dfba0ffdfcc28881b26bb8 +size 614460 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..3b7e47633ea55f3a745db11469ef9ce9ae334771 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f855cdb699fd79990d871c9e03c87c023aedb988e3d083a32c901ef4b9db0bc9 +size 1424446005 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b4df802262b7f5abe7f1602f12f15edc7a12dc7b --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39b5bd88d4c16fcf6dec15569ed42854ab4cfc66719b7d017443d43c67a0e74a +size 1401881227 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..39fe1aa0b8258a5177b632da692a46fb8f431e89 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f964ad1dfee06269bb03459b5d2442761c94c3fd778debfc9f7d35d7f763b487 +size 1406759189 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..ce9d6b892b412f1ed2d88e7e370acb520db56209 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90bf3b1c95b31c44757ae1bb911494b82b25b19fd4c822b32e452126357cbb4e +size 1401210558 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.pth b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.pth new file mode 100644 index 0000000000000000000000000000000000000000..aad3029700e1ea1cb33285d1aa85bc71bb88a215 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ebfa62daa986c87b350c580ab69173f6912fff579c6397be2f5f01a2b5620cb +size 1901 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..32dd13a78376b3378f6fc169d6890727ff87a177 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28cc079c6372d997e97d69f8ef4393810806e6e226026252010ddaa111196068 +size 614460 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..012a9c436062e9f08f6839faf61be300b1421cae --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7359a6528d4fe1c12124da6e6898867a522a3ae7e99591abfde59bc7fca5cdf8 +size 1424446005 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a2d6b2f9773ccf38ea0e377a37e25d5421171312 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb2f439e3c78418fa5f91773426e825580bac9866724183de70638cd1dafbf0e +size 1401881227 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..36466fe6240d13bfc5d5bd10a13fca9a9ea7b1bf --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22960b20af42e183a139c903291fd2858e1194947ad24c0e30fabffc1d65daba +size 1406759189 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8e6cc2f081e9067cd917255e2f409d965d77062e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee902a832740f546065e96ed609eba342f3dee7192a3a2e74134cf2ae5982b6b +size 1401210558 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.pth b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.pth new file mode 100644 index 0000000000000000000000000000000000000000..92a76f0f3b040efee094daabdd04fc4ecb7c670e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a76c71e02345bb8451425e4c637c86c4cc5f60c71d79309cf5524e46013f7295 +size 1901 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..66011304b26c46b7b09a41507e0e1dd95857e091 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db8d58cac82de1b59784174a2f330343605818474199d0d483a81e31c5ffff8f +size 614460 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b348a390acee8788ddf021373438e7d169c41f11 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:470b234eaab9455c9de4a2d9e60b2d9e13bef449307905ae6352c47b844c27e1 +size 1424446005 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..303772a0a3e6c9e21f867e62f27e0439eadbd928 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:012a94e743748762eca4499ba196e6c6e007e68ddc8cee2b47f5f274b43b0291 +size 1401881227 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..09d512b8f165dd47ee4b1acf09995200e5555dfb --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:765808b1cccbb1085a7fa7617128a8d418e4a9acfcb3221941f3824e56a6a364 +size 1406759189 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..dbbd6d4581f6914728727870b3a2b9f0e7df1075 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fa33a305ed81b5b4c5601124ff62c2ed40fefb991c5358336197237b78c65ab +size 1401210558 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.pth b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.pth new file mode 100644 index 0000000000000000000000000000000000000000..6eed4d2ddbaaaa8d1ffb666f806efbca131032aa --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06ff7fb99b3daabe3678198e33994011726c066e03d570d82b78992320415261 +size 1901 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..90dd0b5d7407b339c34a307097677a7eaa21f0b0 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec67094e0fbd8ae949a3dac91e868acd9fb0d88269ede828a89aede31fb14f41 +size 1192023 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..584e1707f39be88271e14d4a084bdb4b859c3410 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e54464d63ad6d15d34074f462416955585db37b7076da7e60ffe10ab51085e7d +size 1599501209 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..49b61e021b1398005658a43bc0ede64cae3213bd --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b94c5c0dacbaec9290a51104015f3c1b9795b09d81b5af6d37f2a4116dee2a60 +size 1598334114 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..117e2b8859d8042c8e52c3d605dfb65b4247e2ab --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:695277b11e9e7830e3d1f78ed89a554411e3d44f3349d13108ec33d4b6bc3c7b +size 1598358322 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..95de703ce0a48ce94871af1cac22b72f45f1c37e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00ad769262e362da35b1462941902fccd5c00bb0ca274e4c42d257137aca54b3 +size 1598356809 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..0f93ca35869387775ba5e069939a6d290289c71d --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0359e801e49d2b28d48496599fd5ea64422c45f22eeb4302886652c20ab49b1 +size 1387307 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b54f67ef39c39785ad4a0300026be9a68a9f911a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f25c5a398cdc9afdea56e3c570a5666a1ddc8fd7b61dac563a026cb58a91a9 +size 351856217 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..828a3e6eeacd41fc4a56663dbbabf819854078f1 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:012cbc0f01b53ce7abfcc6d36879d20d26092a91cb073ade54543ed5f5d62534 +size 352173823 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9c3472eb8119eacd12305ec99b9476dd58caf6c2 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6f2fc58c452a3d7ee60753d909a2a5dd608a448358e286f21e114cfca3a5608 +size 352173823 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..600dfc8d4af3d9efdc4403a11f6c7777b1c10025 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.fake_score_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c22f30169a454708a228f81f5fafc19811e3050bc5a60465f9655076a0d3e2db +size 352211338 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..f1d6cf7638b262bc48c1ce36282a84f2c9915fc6 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f693567c4b957ba49a027fe47d595899f8a821f68a754e1d3adec5766d31a938 +size 614460 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..4275da1c95134fc2c9e01a6e0fd4eea658996ced --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f34ce0ac3e425562ce2da71cea7c20c3d2600e6924132178679770d6689f34d +size 1424446005 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8177c8145ec2205760c20b86f99a71051d334909 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bea654cff805b2ce43602b09709932a84ba3d1d726a386cdd661b5860fd0874 +size 1401881227 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..d8f9a9ce06d3902bd591b417b0278def4ac0ce5c --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdf40456d5de663784263f2ddbbe992ada548b37c8c82fc1cf338ddb5e2a30e1 +size 1406759189 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..3e29a532a4610e0ece70200b5e9e6b5f388c431b --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cabfc22561f150e729183cdae2ccf8539a07728453b5832d9a8f0b2043bf016 +size 1401210558 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..e3ea46f15231319ef253956ec9af3d11092c8531 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e1e8c18fa8ced5286dadd1446e850a58a78f751298e0b5f51aab3028e5d7e68 +size 1649349 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b708a9ef60b041a3eb0ccd5041924c4e93158ee7 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5e8866bb0b61f80dcd96bad92fb799f875664d26d20305d428bb4b1ab07acfa +size 2803269738 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9d6011fdafa3dbf91b0218b69e81f242f32a18f5 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c66da20a709fe70b1633d6509262be2daefa9185124cee754f0bf080d5fa75cd +size 2827716077 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..00f4ddaa5e1223fbc0e9df4f3b5cb518678427a9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:453e6b4b0f8f2049eda12d3b309926f3278977368642cbbaf9d126994b49bad7 +size 2827716077 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e3d07a3a340223bfef8ac237ee7adb12b9b82f1c --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.net_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a6b375280db719686bb15613dde3096a6566a22f59e1557496d20fe1b5d37da +size 2811295404 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.pth b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.pth new file mode 100644 index 0000000000000000000000000000000000000000..1eb73bc1d16a219be160dd7be09e2fe36228d59a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0548d00c34d800d0bd0e3395f27252efde74712d987b8730e05e6bfaa7db4ff9 +size 1901 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/config.yaml b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98ccd097267be3949098f44093a2386451ebd6b8 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/config.yaml @@ -0,0 +1,282 @@ +dataloader_train: + _target_: + batch_size: '8' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: 'True' + neg_text_emb_path: null + num_workers: '2' + raw_audio_fps: '25.0' + raw_audio_num_frames: '81' + raw_audio_sample_rate: '16000' + use_ref_sequence: 'True' +dataloader_val: + _target_: + batch_size: '1' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: 'False' + neg_text_emb_path: null + num_workers: '2' + use_ref_sequence: 'True' +eval: + max_ckpt: '100000000' + min_ckpt: '0' + num_samples: '50000' + samples_dir: samples + save_images: 'False' +log_config: + group: omniavatar_sf_audiofix + name: sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100 + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + add_teacher_to_fsdp_dict: 'True' + center_reward: 'False' + clamp_reward: null + context_noise: '0.0' + ddp_find_unused_parameters: 'True' + device: cuda + discriminator: + _target_: + all_res: + - '32' + - '16' + - '8' + feature_indices: '{0, 1, 2}' + in_channels: '256' + discriminator_optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: '0.0001' + model: null + optim_type: adamw + weight_decay: '0.01' + discriminator_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + enable_gradient_in_rollout: 'True' + enable_preprocessors: 'True' + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: '65' + merge_lora: 'False' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: 'True' + fake_score_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 3e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + fsdp_meta_init: 'False' + gan_loss_weight_gen: '0' + gan_r1_reg_alpha: '0.1' + gan_r1_reg_weight: '0.0' + gan_use_same_t_noise: 'False' + grad_scaler_enabled: 'False' + grad_scaler_growth_interval: '2000' + grad_scaler_init_scale: '65536.0' + guidance_scale: '4.5' + input_shape: + - '16' + - '21' + - '64' + - '64' + last_step_only: 'False' + load_student_weights: 'False' + net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: '3' + in_dim: '65' + local_attn_size: '9' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: '3' + total_num_frames: '21' + use_audio: 'True' + use_dynamic_rope: 'True' + net_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + net_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: '' + pretrained_student_net_path: '' + reward: + audio_sample_rate: '16000' + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: 'True' + input_fps: '25.0' + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: '15' + reward_beta: '2' + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: 'True' + sample_t_cfg: + log_t_df: '0.01' + max_t: '0.999' + min_t: '0.001' + shift: '5.0' + t_list: + - '0.999' + - '0.833' + - '0.0' + time_dist_type: shifted + train_p_mean: '-1.1' + train_p_std: '2.0' + save_reward_debug_video: 'False' + skip_layers: null + start_gradient_frame: '0' + student_sample_steps: '2' + student_sample_type: sde + student_update_freq: '5' + teacher: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: 'True' + timestep_cfg: + enabled: 'True' + t_hi: '0.882' + t_lo: '0.556' + use_ema: 'False' +model_class: + _target_: + config: null +trainer: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: '0.9999' + ema_halflife_kimg: '500' + ema_rampup_ratio: '0.05' + gamma: '16.97' + start_iter: '0' + type: constant + gpu_stats: + _target_: + every_n: '100' + grad_clip: + _target_: + grad_norm: '10.0' + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: '100' + wandb: + _target_: + fps: '25' + sample_logging_iter: '100' + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_100iter/checkpoints/0000100.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints + use_s3: 'False' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: 'False' + fsdp: 'True' + fsdp_cpu_offload: 'False' + fsdp_min_num_params: '10000000' + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: '2' + logging_iter: '1' + max_iter: '5000' + offload_module_in_decoding: 'False' + resume: 'False' + save_ckpt_iter: '100' + seed: '0' + skip_initial_validation: 'True' + tf32_enabled: 'True' + val_seed: null + validation_iter: '100' + visualize_teacher: 'False' diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/debug-internal.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4e2bc57ab266dbaf76bdf43543f8df58b6d0fbd2 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/debug-internal.log @@ -0,0 +1,7 @@ +{"time":"2026-04-25T14:26:02.799847511+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-04-25T14:26:03.186449198+09:00","level":"INFO","msg":"stream: created new stream","id":"3fi9xe8g"} +{"time":"2026-04-25T14:26:03.186497211+09:00","level":"INFO","msg":"handler: started","stream_id":"3fi9xe8g"} +{"time":"2026-04-25T14:26:03.186565827+09:00","level":"INFO","msg":"stream: started","id":"3fi9xe8g"} +{"time":"2026-04-25T14:26:03.186588125+09:00","level":"INFO","msg":"sender: started","stream_id":"3fi9xe8g"} +{"time":"2026-04-25T14:26:03.186587608+09:00","level":"INFO","msg":"writer: started","stream_id":"3fi9xe8g"} +{"time":"2026-04-26T01:28:38.020968115+09:00","level":"INFO","msg":"stream: closing","id":"3fi9xe8g"} diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/debug.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..60ac9f2e459e930481062941e7a4246d4df9506a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/debug.log @@ -0,0 +1,21 @@ +2026-04-25 14:26:02,551 INFO MainThread:953349 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-04-25 14:26:02,551 INFO MainThread:953349 [wandb_setup.py:_flush():81] Configure stats pid to 953349 +2026-04-25 14:26:02,551 INFO MainThread:953349 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-25 14:26:02,551 INFO MainThread:953349 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/logs/debug.log +2026-04-25 14:26:02,551 INFO MainThread:953349 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/logs/debug-internal.log +2026-04-25 14:26:02,551 INFO MainThread:953349 [wandb_init.py:init():844] calling init triggers +2026-04-25 14:26:02,551 INFO MainThread:953349 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_audiofix', 'name': 'sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_100iter/checkpoints/0000100.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '5000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': None, 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': None, 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '9', 'sink_size': '3', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'False', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.833', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '3e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'True', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '2', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-04-25 14:26:02,551 INFO MainThread:953349 [wandb_init.py:init():892] starting backend +2026-04-25 14:26:02,785 INFO MainThread:953349 [wandb_init.py:init():895] sending inform_init request +2026-04-25 14:26:02,797 INFO MainThread:953349 [wandb_init.py:init():903] backend started and connected +2026-04-25 14:26:02,801 INFO MainThread:953349 [wandb_init.py:init():973] updated telemetry +2026-04-25 14:26:02,825 INFO MainThread:953349 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-25 14:26:03,857 INFO MainThread:953349 [wandb_init.py:init():1042] starting run threads in backend +2026-04-25 14:26:04,224 INFO MainThread:953349 [wandb_run.py:_console_start():2524] atexit reg +2026-04-25 14:26:04,225 INFO MainThread:953349 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-04-25 14:26:04,225 INFO MainThread:953349 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-04-25 14:26:04,225 INFO MainThread:953349 [wandb_run.py:_redirect():2465] Redirects installed. +2026-04-25 14:26:04,232 INFO MainThread:953349 [wandb_init.py:init():1082] run started, returning control to user process +2026-04-26 01:28:38,020 INFO wandb-AsyncioManager-main:953349 [service_client.py:_forward_responses():134] Reached EOF. +2026-04-26 01:28:38,021 INFO wandb-AsyncioManager-main:953349 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/config.yaml b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd6ed4a0f6505650a679011f2d1322d1684cda13 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/config.yaml @@ -0,0 +1,389 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + bfyevs0fdc4ubo27edzw6b12a0hspzmh: + args: + - --config=fastgen/configs/experiments/OmniAvatar/config_sf_sink1_window7_redmd_beta2_taew.py + - '-' + - trainer.resume=False + - model.fake_score_optimizer.lr=3e-6 + - log_config.group=omniavatar_sf_audiofix + - log_config.name=sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100 + - log_config.project=OmniAvatar-FastGen + - log_config.wandb_entity=paulhcho + - model.net.sink_size=3 + - model.net.local_attn_size=9 + codePath: train.py + codePathLocal: train.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.9" + disk: + /: + total: "1356758433792" + used: "1275297685504" + email: paul.hyunbin@gmail.com + executable: /home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12 + git: + commit: 31752c6328ad798bdbb8f20a8372fca5fb35b990 + remote: https://paulcho98:@github.com/paulcho98/FastGen.git + gpu: NVIDIA H200 + gpu_count: 4 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0 + host: main1 + memory: + total: "2163961778176" + os: Linux-5.15.0-151-generic-x86_64-with-glibc2.39 + program: /home/work/.local/hyunbin/FastGen-redmd/train.py + python: CPython 3.12.12 + root: /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100 + startedAt: "2026-04-25T02:54:05.241185Z" + writerId: bfyevs0fdc4ubo27edzw6b12a0hspzmh + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "3": + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.12.12 + "5": 0.25.0 + "6": 5.3.0 + "12": 0.25.0 + "13": linux-x86_64 +dataloader_train: + value: + _target_: + batch_size: "8" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: "True" + neg_text_emb_path: null + num_workers: "2" + raw_audio_fps: "25.0" + raw_audio_num_frames: "81" + raw_audio_sample_rate: "16000" + use_ref_sequence: "True" +dataloader_val: + value: + _target_: + batch_size: "1" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: "False" + neg_text_emb_path: null + num_workers: "2" + use_ref_sequence: "True" +eval: + value: + max_ckpt: "100000000" + min_ckpt: "0" + num_samples: "50000" + samples_dir: samples + save_images: "False" +log_config: + value: + group: omniavatar_sf_audiofix + name: sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100 + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + value: + add_teacher_to_fsdp_dict: "True" + center_reward: "False" + clamp_reward: null + context_noise: "0.0" + ddp_find_unused_parameters: "True" + device: cuda + discriminator: + _target_: + all_res: + - "32" + - "16" + - "8" + feature_indices: '{0, 1, 2}' + in_channels: "256" + discriminator_optimizer: + _target_: + betas: + - "0.9" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "0.0001" + model: null + optim_type: adamw + weight_decay: "0.01" + discriminator_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + enable_gradient_in_rollout: "True" + enable_preprocessors: "True" + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: "65" + merge_lora: "False" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: "True" + fake_score_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "3e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + fsdp_meta_init: "False" + gan_loss_weight_gen: "0" + gan_r1_reg_alpha: "0.1" + gan_r1_reg_weight: "0.0" + gan_use_same_t_noise: "False" + grad_scaler_enabled: "False" + grad_scaler_growth_interval: "2000" + grad_scaler_init_scale: "65536.0" + guidance_scale: "4.5" + input_shape: + - "16" + - "21" + - "64" + - "64" + last_step_only: "False" + load_student_weights: "False" + net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: "3" + in_dim: "65" + local_attn_size: "9" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: "3" + total_num_frames: "21" + use_audio: "True" + use_dynamic_rope: "True" + net_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + net_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: "" + pretrained_student_net_path: "" + reward: + audio_sample_rate: "16000" + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: "True" + input_fps: "25.0" + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: "15" + reward_beta: "2" + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: "True" + sample_t_cfg: + log_t_df: "0.01" + max_t: "0.999" + min_t: "0.001" + shift: "5.0" + t_list: + - "0.999" + - "0.833" + - "0.0" + time_dist_type: shifted + train_p_mean: "-1.1" + train_p_std: "2.0" + save_reward_debug_video: "False" + skip_layers: null + start_gradient_frame: "0" + student_sample_steps: "2" + student_sample_type: sde + student_update_freq: "5" + teacher: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: "True" + timestep_cfg: + enabled: "True" + t_hi: "0.882" + t_lo: "0.556" + use_ema: "False" +model_class: + value: + _target_: + config: null +trainer: + value: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: "0.9999" + ema_halflife_kimg: "500" + ema_rampup_ratio: "0.05" + gamma: "16.97" + start_iter: "0" + type: constant + gpu_stats: + _target_: + every_n: "100" + grad_clip: + _target_: + grad_norm: "10.0" + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: "100" + wandb: + _target_: + fps: "25" + sample_logging_iter: "100" + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_100iter/checkpoints/0000100.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints + use_s3: "False" + cudnn: + benchmark: "True" + deterministic: "False" + ddp: "False" + fsdp: "True" + fsdp_cpu_offload: "False" + fsdp_min_num_params: "10000000" + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: "2" + logging_iter: "1" + max_iter: "5000" + offload_module_in_decoding: "False" + resume: "False" + save_ckpt_iter: "100" + seed: "0" + skip_initial_validation: "True" + tf32_enabled: "True" + val_seed: null + validation_iter: "100" + visualize_teacher: "False" diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23f2f377ccf704bcaa09a4c9b1a9b5151ab73aac --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47403c800591fce4a99b38622b7607baeec11f4e025df3962d292acf880fba6c +size 192248 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/media/videos/train_media/student/generation_100_04a9b61f163a19decce9.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/media/videos/train_media/student/generation_100_04a9b61f163a19decce9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9c9cd1dafe014aef74d70823bd6e17a310abbda3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/media/videos/train_media/student/generation_100_04a9b61f163a19decce9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04a9b61f163a19decce96a0fd80161cd6c80a689cf81620d9c2d832ab2556052 +size 257128 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/output.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..88182632a092ea0310647a394538d81dff684565 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/output.log @@ -0,0 +1,2214 @@ +[Apr 25, 2026 - 11:54:06 | INFO | fastgen.callbacks.wandb:init_wandb:259 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[Apr 25, 2026 - 11:54:06 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[Apr 25, 2026 - 11:54:06 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[Apr 25, 2026 - 11:54:06 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[Apr 25, 2026 - 11:54:06 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[Apr 25, 2026 - 11:54:06 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[Apr 25, 2026 - 11:54:06 | INFO | fastgen.trainer:run:77 ] Starting training +[Apr 25, 2026 - 11:54:06 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[Apr 25, 2026 - 11:54:06 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_100iter/checkpoints/0000100.pth +[Apr 25, 2026 - 11:54:33 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict... +[Apr 25, 2026 - 11:54:45 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: +[Apr 25, 2026 - 11:54:45 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_100iter/checkpoints/0000100.pth at iteration 100 +[Apr 25, 2026 - 11:54:45 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 100. +[Apr 25, 2026 - 11:54:45 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[Apr 25, 2026 - 11:55:51 | INFO | fastgen.methods.model:on_train_begin:296 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[Apr 25, 2026 - 11:55:51 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[Apr 25, 2026 - 11:55:51 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[Apr 25, 2026 - 11:55:51 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[Apr 25, 2026 - 11:55:51 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)... +[Apr 25, 2026 - 11:55:51 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2166 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[Apr 25, 2026 - 11:55:54 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[Apr 25, 2026 - 11:55:54 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 2.9s +[Apr 25, 2026 - 11:55:54 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.60B params)... +[Apr 25, 2026 - 11:55:54 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:774 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[Apr 25, 2026 - 11:55:56 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[Apr 25, 2026 - 11:55:57 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 2.6s +[Apr 25, 2026 - 11:55:57 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[Apr 25, 2026 - 11:55:57 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:774 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[Apr 25, 2026 - 11:56:09 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[Apr 25, 2026 - 11:56:10 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 13.0s +[Apr 25, 2026 - 11:56:10 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[Apr 25, 2026 - 11:56:10 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[Apr 25, 2026 - 11:56:10 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[Apr 25, 2026 - 11:56:10 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 0. +[Apr 25, 2026 - 11:56:10 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader... +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:362 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1 +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1 +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingReDMD) has 1596.36 M trainable and 17325.46 M total params (logical). +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.42 M trainable and 4364.06 M total params LOCAL on rank 0. +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.14 M trainable and 4362.26 M total params LOCAL on rank 1. +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.14 M trainable and 4362.26 M total params LOCAL on rank 2. +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.14 M trainable and 4362.26 M total params LOCAL on rank 3. +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.68 M trainable and 376.68 M total params LOCAL on rank 0. +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 1. +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 2. +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 3. +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 174.98 M trainable and 1596.36 M total params (logical). +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.30 M total params LOCAL on rank 0. +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 1. +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 2. +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 3. +[Apr 25, 2026 - 11:56:15 | INFO | fastgen.trainer:run:174 ] iter_start: 0 +[Apr 25, 2026 - 11:57:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.567289 38.600391 38.537437 +peak_gpu_mem_gb 60.567137 60.567137 60.567137 +peak_gpu_mem_reserved_gb 62.595703 62.595703 62.595703 +util 96.000000 99.000000 92.000000 +[Apr 25, 2026 - 11:57:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 1 : data loading time 1.13 +[Apr 25, 2026 - 11:57:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 1 : avg forward pass time 11.17 +[Apr 25, 2026 - 11:57:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 1 : backward pass time 11.54 +[Apr 25, 2026 - 11:57:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 1 : optimizer step time 1.07 +[Apr 25, 2026 - 11:57:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 1-------------------- +[Apr 25, 2026 - 11:57:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0171 iter count: 1.0 +[Apr 25, 2026 - 11:57:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0171 iter count: 1.0 +[Apr 25, 2026 - 11:57:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 11:57:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 11:57:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.742749 38.776199 38.712746 +peak_gpu_mem_gb 62.354346 62.354346 62.354346 +peak_gpu_mem_reserved_gb 64.042969 64.042969 64.042969 +util 98.250000 100.000000 96.000000 +[Apr 25, 2026 - 11:57:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 2 : avg iteration time 49.95 seconds +[Apr 25, 2026 - 11:57:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 2 : data loading time 0.00 +[Apr 25, 2026 - 11:57:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 2 : avg forward pass time 12.91 +[Apr 25, 2026 - 11:57:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 2 : backward pass time 11.57 +[Apr 25, 2026 - 11:57:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 2 : optimizer step time 0.98 +[Apr 25, 2026 - 11:57:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 2-------------------- +[Apr 25, 2026 - 11:57:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0139 iter count: 1.0 +[Apr 25, 2026 - 11:57:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[Apr 25, 2026 - 11:57:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 11:57:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 11:58:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.742567 38.775154 38.712807 +peak_gpu_mem_gb 62.354346 62.354346 62.354346 +peak_gpu_mem_reserved_gb 64.042969 64.042969 64.042969 +util 97.250000 100.000000 94.000000 +[Apr 25, 2026 - 11:58:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 3 : avg iteration time 52.91 seconds +[Apr 25, 2026 - 11:58:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 3 : data loading time 0.00 +[Apr 25, 2026 - 11:58:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 3 : avg forward pass time 14.38 +[Apr 25, 2026 - 11:58:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 3 : backward pass time 11.59 +[Apr 25, 2026 - 11:58:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 3 : optimizer step time 0.96 +[Apr 25, 2026 - 11:58:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 3-------------------- +[Apr 25, 2026 - 11:58:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0120 iter count: 1.0 +[Apr 25, 2026 - 11:58:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[Apr 25, 2026 - 11:58:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 11:58:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 11:59:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.742831 38.775269 38.713703 +peak_gpu_mem_gb 62.354346 62.354346 62.354346 +peak_gpu_mem_reserved_gb 64.062500 64.062500 64.062500 +util 96.000000 100.000000 92.000000 +[Apr 25, 2026 - 11:59:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 4 : avg iteration time 47.16 seconds +[Apr 25, 2026 - 11:59:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 4 : data loading time 0.00 +[Apr 25, 2026 - 11:59:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 4 : avg forward pass time 11.51 +[Apr 25, 2026 - 11:59:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 4 : backward pass time 11.56 +[Apr 25, 2026 - 11:59:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 4 : optimizer step time 0.99 +[Apr 25, 2026 - 11:59:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 4-------------------- +[Apr 25, 2026 - 11:59:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0177 iter count: 1.0 +[Apr 25, 2026 - 11:59:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0177 iter count: 1.0 +[Apr 25, 2026 - 11:59:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 11:59:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.805329 38.836208 38.776745 +peak_gpu_mem_gb 126.559253 126.559253 126.559253 +peak_gpu_mem_reserved_gb 138.166016 138.166016 138.166016 +util 58.750000 63.000000 54.000000 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 5 : avg iteration time 180.08 seconds +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 5 : data loading time 0.00 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 5 : avg forward pass time 80.08 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 5 : backward pass time 10.35 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 5 : optimizer step time 0.41 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 5-------------------- +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 19016.2590 iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6166 iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 19016.2590iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.2858iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5293 iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.8955 iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 34924.1602iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.8823 iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 975849.1250iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6166iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.7891 iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1955.1912iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.9429 iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2659.1538iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.4437 iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 979.7997iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 1.9510 iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 49.5002 iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0165 iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:02:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 12:03:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.805403 38.836449 38.776756 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.867676 67.869141 67.867188 +util 98.750000 100.000000 97.000000 +[Apr 25, 2026 - 12:03:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 6 : avg iteration time 53.27 seconds +[Apr 25, 2026 - 12:03:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 6 : data loading time 0.00 +[Apr 25, 2026 - 12:03:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 6 : avg forward pass time 14.60 +[Apr 25, 2026 - 12:03:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 6 : backward pass time 11.54 +[Apr 25, 2026 - 12:03:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 6 : optimizer step time 0.96 +[Apr 25, 2026 - 12:03:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 6-------------------- +[Apr 25, 2026 - 12:03:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 12:03:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 12:03:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:03:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:04:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.805998 38.837959 38.776764 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.867676 67.869141 67.867188 +util 96.000000 99.000000 92.000000 +[Apr 25, 2026 - 12:04:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 7 : avg iteration time 52.95 seconds +[Apr 25, 2026 - 12:04:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 7 : data loading time 0.00 +[Apr 25, 2026 - 12:04:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 7 : avg forward pass time 14.41 +[Apr 25, 2026 - 12:04:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 7 : backward pass time 11.57 +[Apr 25, 2026 - 12:04:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 7 : optimizer step time 0.99 +[Apr 25, 2026 - 12:04:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 7-------------------- +[Apr 25, 2026 - 12:04:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0125 iter count: 1.0 +[Apr 25, 2026 - 12:04:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[Apr 25, 2026 - 12:04:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:04:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:05:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.806013 38.837994 38.776764 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.809082 67.810547 67.808594 +util 95.500000 100.000000 91.000000 +[Apr 25, 2026 - 12:05:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 8 : avg iteration time 52.79 seconds +[Apr 25, 2026 - 12:05:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 8 : data loading time 0.00 +[Apr 25, 2026 - 12:05:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 8 : avg forward pass time 14.36 +[Apr 25, 2026 - 12:05:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 8 : backward pass time 11.54 +[Apr 25, 2026 - 12:05:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 8 : optimizer step time 0.97 +[Apr 25, 2026 - 12:05:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 8-------------------- +[Apr 25, 2026 - 12:05:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 12:05:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 12:05:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:05:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:06:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.806096 38.838261 38.776783 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.828613 67.830078 67.828125 +util 97.000000 99.000000 94.000000 +[Apr 25, 2026 - 12:06:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 9 : avg iteration time 46.96 seconds +[Apr 25, 2026 - 12:06:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 9 : data loading time 0.00 +[Apr 25, 2026 - 12:06:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 9 : avg forward pass time 11.45 +[Apr 25, 2026 - 12:06:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 9 : backward pass time 11.54 +[Apr 25, 2026 - 12:06:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 9 : optimizer step time 0.97 +[Apr 25, 2026 - 12:06:01 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 9-------------------- +[Apr 25, 2026 - 12:06:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0153 iter count: 1.0 +[Apr 25, 2026 - 12:06:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0153 iter count: 1.0 +[Apr 25, 2026 - 12:06:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:06:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.806305 38.837208 38.777405 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.094238 133.095703 133.093750 +util 73.250000 79.000000 68.000000 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 10 : avg iteration time 175.61 seconds +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 10 : data loading time 0.00 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 10 : avg forward pass time 78.55 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 10 : backward pass time 9.14 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 10 : optimizer step time 0.32 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 10-------------------- +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 15259.6733 iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6155 iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 15259.6733iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.0561iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6559 iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.1237 iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 24889.1523iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.7132 iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 208449.0312iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6155iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.5930 iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 72107.8906iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.2817 iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 708.7341iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.1385 iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 3932.5615iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.0975 iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 197800.4531iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0156 iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:08:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 12:09:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.806546 38.838051 38.777462 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.940918 67.986328 67.925781 +util 97.000000 99.000000 91.000000 +[Apr 25, 2026 - 12:09:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 11 : avg iteration time 51.33 seconds +[Apr 25, 2026 - 12:09:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 11 : data loading time 0.00 +[Apr 25, 2026 - 12:09:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 11 : avg forward pass time 13.62 +[Apr 25, 2026 - 12:09:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 11 : backward pass time 11.54 +[Apr 25, 2026 - 12:09:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 11 : optimizer step time 0.97 +[Apr 25, 2026 - 12:09:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 11-------------------- +[Apr 25, 2026 - 12:09:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0137 iter count: 1.0 +[Apr 25, 2026 - 12:09:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0137 iter count: 1.0 +[Apr 25, 2026 - 12:09:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:09:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:10:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.806753 38.838802 38.777424 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.901855 67.947266 67.886719 +util 97.500000 100.000000 92.000000 +[Apr 25, 2026 - 12:10:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 12 : avg iteration time 52.80 seconds +[Apr 25, 2026 - 12:10:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 12 : data loading time 0.00 +[Apr 25, 2026 - 12:10:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 12 : avg forward pass time 14.37 +[Apr 25, 2026 - 12:10:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 12 : backward pass time 11.55 +[Apr 25, 2026 - 12:10:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 12 : optimizer step time 0.99 +[Apr 25, 2026 - 12:10:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 12-------------------- +[Apr 25, 2026 - 12:10:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0113 iter count: 1.0 +[Apr 25, 2026 - 12:10:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[Apr 25, 2026 - 12:10:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:10:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:11:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.80676 38.838825 38.777466 +peak_gpu_mem_gb 65.49830 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.89209 67.906250 67.849609 +util 97.00000 99.000000 94.000000 +[Apr 25, 2026 - 12:11:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 13 : avg iteration time 52.86 seconds +[Apr 25, 2026 - 12:11:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 13 : data loading time 0.00 +[Apr 25, 2026 - 12:11:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 13 : avg forward pass time 14.39 +[Apr 25, 2026 - 12:11:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 13 : backward pass time 11.53 +[Apr 25, 2026 - 12:11:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 13 : optimizer step time 0.99 +[Apr 25, 2026 - 12:11:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 13-------------------- +[Apr 25, 2026 - 12:11:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0101 iter count: 1.0 +[Apr 25, 2026 - 12:11:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[Apr 25, 2026 - 12:11:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:11:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:12:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.806673 38.838444 38.777473 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.921387 67.925781 67.908203 +util 97.250000 100.000000 94.000000 +[Apr 25, 2026 - 12:12:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 14 : avg iteration time 49.88 seconds +[Apr 25, 2026 - 12:12:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 14 : data loading time 0.00 +[Apr 25, 2026 - 12:12:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 14 : avg forward pass time 12.91 +[Apr 25, 2026 - 12:12:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 14 : backward pass time 11.54 +[Apr 25, 2026 - 12:12:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 14 : optimizer step time 0.97 +[Apr 25, 2026 - 12:12:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 14-------------------- +[Apr 25, 2026 - 12:12:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0145 iter count: 1.0 +[Apr 25, 2026 - 12:12:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0145 iter count: 1.0 +[Apr 25, 2026 - 12:12:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:12:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.806445 38.837509 38.777470 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.149414 133.193359 133.134766 +util 72.750000 77.000000 69.000000 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 15 : avg iteration time 169.23 seconds +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 15 : data loading time 0.00 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 15 : avg forward pass time 75.20 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 15 : backward pass time 9.13 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 15 : optimizer step time 0.32 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 15-------------------- +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 8898.3840 iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5955 iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 8898.3840iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.4692iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6644 iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 5.9423 iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 13514.9854iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.7762 iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 145030.1562iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5955iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.1875 iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 4337.6489iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.3457 iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 5951.7842iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.1390 iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 72.1016 iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.2541 iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 4955.1206iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:15:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 12:16:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.806951 38.837776 38.777470 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.950684 67.964844 67.908203 +util 96.500000 99.000000 93.000000 +[Apr 25, 2026 - 12:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 16 : avg iteration time 48.36 seconds +[Apr 25, 2026 - 12:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 16 : data loading time 0.00 +[Apr 25, 2026 - 12:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 16 : avg forward pass time 12.15 +[Apr 25, 2026 - 12:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 16 : backward pass time 11.52 +[Apr 25, 2026 - 12:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 16 : optimizer step time 0.99 +[Apr 25, 2026 - 12:16:01 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 16-------------------- +[Apr 25, 2026 - 12:16:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0136 iter count: 1.0 +[Apr 25, 2026 - 12:16:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[Apr 25, 2026 - 12:16:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:16:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:16:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.807189 38.838661 38.777523 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.701660 67.849609 67.652344 +util 99.000000 99.000000 99.000000 +[Apr 25, 2026 - 12:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 17 : avg iteration time 49.87 seconds +[Apr 25, 2026 - 12:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 17 : data loading time 0.00 +[Apr 25, 2026 - 12:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 17 : avg forward pass time 12.90 +[Apr 25, 2026 - 12:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 17 : backward pass time 11.53 +[Apr 25, 2026 - 12:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 17 : optimizer step time 0.98 +[Apr 25, 2026 - 12:16:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 17-------------------- +[Apr 25, 2026 - 12:16:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0109 iter count: 1.0 +[Apr 25, 2026 - 12:16:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[Apr 25, 2026 - 12:16:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:16:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:17:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.807837 38.839527 38.778370 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.862793 67.925781 67.673828 +util 96.500000 99.000000 93.000000 +[Apr 25, 2026 - 12:17:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 18 : avg iteration time 49.93 seconds +[Apr 25, 2026 - 12:17:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 18 : data loading time 0.00 +[Apr 25, 2026 - 12:17:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 18 : avg forward pass time 12.92 +[Apr 25, 2026 - 12:17:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 18 : backward pass time 11.54 +[Apr 25, 2026 - 12:17:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 18 : optimizer step time 0.99 +[Apr 25, 2026 - 12:17:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 18-------------------- +[Apr 25, 2026 - 12:17:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 12:17:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 12:17:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:17:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:18:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808171 38.840008 38.779263 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.721191 67.869141 67.671875 +util 97.250000 100.000000 91.000000 +[Apr 25, 2026 - 12:18:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 19 : avg iteration time 49.88 seconds +[Apr 25, 2026 - 12:18:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 19 : data loading time 0.00 +[Apr 25, 2026 - 12:18:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 19 : avg forward pass time 12.92 +[Apr 25, 2026 - 12:18:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 19 : backward pass time 11.53 +[Apr 25, 2026 - 12:18:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 19 : optimizer step time 0.96 +[Apr 25, 2026 - 12:18:31 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 19-------------------- +[Apr 25, 2026 - 12:18:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 12:18:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 12:18:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:18:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808183 38.839821 38.779354 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.125488 133.156250 133.115234 +util 70.750000 76.000000 65.000000 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 20 : avg iteration time 175.04 seconds +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 20 : data loading time 0.00 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 20 : avg forward pass time 77.84 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 20 : backward pass time 9.12 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 20 : optimizer step time 0.32 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 20-------------------- +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 25018.3633 iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6838 iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 25018.3633iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.7754iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.6915 iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.7304 iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 48923.8867iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 29.4570iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 701379.1875iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6838iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.0193 iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 22892.1504iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.3330 iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 785.2831iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.1614 iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 30416.2129iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.6138 iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 555456.0625iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:21:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 12:22:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808180 38.839821 38.779308 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.911621 67.927734 67.906250 +util 95.000000 99.000000 91.000000 +[Apr 25, 2026 - 12:22:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 21 : avg iteration time 48.46 seconds +[Apr 25, 2026 - 12:22:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 21 : data loading time 0.00 +[Apr 25, 2026 - 12:22:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 21 : avg forward pass time 12.21 +[Apr 25, 2026 - 12:22:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 21 : backward pass time 11.54 +[Apr 25, 2026 - 12:22:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 21 : optimizer step time 0.98 +[Apr 25, 2026 - 12:22:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 21-------------------- +[Apr 25, 2026 - 12:22:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 12:22:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 12:22:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:22:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:23:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808182 38.839813 38.779312 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.862793 67.966797 67.828125 +util 96.500000 99.000000 93.000000 +[Apr 25, 2026 - 12:23:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 22 : avg iteration time 49.82 seconds +[Apr 25, 2026 - 12:23:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 22 : data loading time 0.00 +[Apr 25, 2026 - 12:23:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 22 : avg forward pass time 12.88 +[Apr 25, 2026 - 12:23:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 22 : backward pass time 11.54 +[Apr 25, 2026 - 12:23:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 22 : optimizer step time 0.99 +[Apr 25, 2026 - 12:23:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 22-------------------- +[Apr 25, 2026 - 12:23:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0103 iter count: 1.0 +[Apr 25, 2026 - 12:23:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[Apr 25, 2026 - 12:23:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:23:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:23:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808172 38.839798 38.779312 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.853027 67.927734 67.828125 +util 97.000000 99.000000 91.000000 +[Apr 25, 2026 - 12:23:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 23 : avg iteration time 49.88 seconds +[Apr 25, 2026 - 12:23:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 23 : data loading time 0.00 +[Apr 25, 2026 - 12:23:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 23 : avg forward pass time 12.91 +[Apr 25, 2026 - 12:23:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 23 : backward pass time 11.55 +[Apr 25, 2026 - 12:23:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 23 : optimizer step time 0.96 +[Apr 25, 2026 - 12:23:54 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 23-------------------- +[Apr 25, 2026 - 12:23:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0119 iter count: 1.0 +[Apr 25, 2026 - 12:23:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 25, 2026 - 12:23:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:23:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:24:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808174 38.839802 38.779312 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.911621 67.986328 67.886719 +util 96.250000 99.000000 93.000000 +[Apr 25, 2026 - 12:24:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 24 : avg iteration time 49.84 seconds +[Apr 25, 2026 - 12:24:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 24 : data loading time 0.00 +[Apr 25, 2026 - 12:24:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 24 : avg forward pass time 12.89 +[Apr 25, 2026 - 12:24:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 24 : backward pass time 11.55 +[Apr 25, 2026 - 12:24:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 24 : optimizer step time 0.96 +[Apr 25, 2026 - 12:24:44 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 24-------------------- +[Apr 25, 2026 - 12:24:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 12:24:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 12:24:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:24:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808261 38.840118 38.779320 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.152832 133.191406 133.037109 +util 68.250000 71.000000 64.000000 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 25 : avg iteration time 169.06 seconds +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 25 : data loading time 0.00 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 25 : avg forward pass time 74.68 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 25 : backward pass time 9.13 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 25 : optimizer step time 0.32 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 25-------------------- +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 18258.8547 iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5946 iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 18258.8547iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.2284iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5751 iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0404 iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 44661.2930iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.1590 iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1303692.7500iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5946iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 1.6244 iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 25.7586 iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 0.5751 iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 3.1590 iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.5040 iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1105.3438iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.5507 iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 8967.7217iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:27:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 12:28:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808270 38.840111 38.779320 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.004395 68.123047 67.964844 +util 96.000000 99.000000 92.000000 +[Apr 25, 2026 - 12:28:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 26 : avg iteration time 48.42 seconds +[Apr 25, 2026 - 12:28:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 26 : data loading time 0.00 +[Apr 25, 2026 - 12:28:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 26 : avg forward pass time 12.18 +[Apr 25, 2026 - 12:28:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 26 : backward pass time 11.54 +[Apr 25, 2026 - 12:28:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 26 : optimizer step time 0.97 +[Apr 25, 2026 - 12:28:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 26-------------------- +[Apr 25, 2026 - 12:28:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0127 iter count: 1.0 +[Apr 25, 2026 - 12:28:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[Apr 25, 2026 - 12:28:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:28:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:29:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808272 38.840111 38.779324 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.887207 68.005859 67.847656 +util 98.750000 100.000000 97.000000 +[Apr 25, 2026 - 12:29:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 27 : avg iteration time 52.86 seconds +[Apr 25, 2026 - 12:29:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 27 : data loading time 0.00 +[Apr 25, 2026 - 12:29:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 27 : avg forward pass time 14.38 +[Apr 25, 2026 - 12:29:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 27 : backward pass time 11.57 +[Apr 25, 2026 - 12:29:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 27 : optimizer step time 0.98 +[Apr 25, 2026 - 12:29:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 27-------------------- +[Apr 25, 2026 - 12:29:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0108 iter count: 1.0 +[Apr 25, 2026 - 12:29:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 25, 2026 - 12:29:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:29:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:30:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808273 38.840115 38.779324 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.955566 68.044922 67.925781 +util 97.000000 100.000000 94.000000 +[Apr 25, 2026 - 12:30:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 28 : avg iteration time 52.76 seconds +[Apr 25, 2026 - 12:30:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 28 : data loading time 0.00 +[Apr 25, 2026 - 12:30:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 28 : avg forward pass time 14.35 +[Apr 25, 2026 - 12:30:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 28 : backward pass time 11.54 +[Apr 25, 2026 - 12:30:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 28 : optimizer step time 0.99 +[Apr 25, 2026 - 12:30:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 28-------------------- +[Apr 25, 2026 - 12:30:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0096 iter count: 1.0 +[Apr 25, 2026 - 12:30:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[Apr 25, 2026 - 12:30:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:30:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:30:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808333 38.840363 38.779324 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.965332 68.083984 67.925781 +util 99.000000 99.000000 99.000000 +[Apr 25, 2026 - 12:30:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 29 : avg iteration time 49.86 seconds +[Apr 25, 2026 - 12:30:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 29 : data loading time 0.00 +[Apr 25, 2026 - 12:30:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 29 : avg forward pass time 12.91 +[Apr 25, 2026 - 12:30:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 29 : backward pass time 11.54 +[Apr 25, 2026 - 12:30:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 29 : optimizer step time 0.97 +[Apr 25, 2026 - 12:30:57 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 29-------------------- +[Apr 25, 2026 - 12:30:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 25, 2026 - 12:30:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 25, 2026 - 12:30:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:30:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.807999 38.838821 38.779354 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.167480 133.191406 133.095703 +util 66.750000 71.000000 62.000000 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 30 : avg iteration time 177.90 seconds +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 30 : data loading time 0.00 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 30 : avg forward pass time 79.10 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 30 : backward pass time 9.13 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 30 : optimizer step time 0.32 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 30-------------------- +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 134222.4038 iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6647 iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 134222.4038iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.8499iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.9234 iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.9643 iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 274189.3750iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 46.8472iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 8273269.0000iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6647iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.1783 iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 31466.4160iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.4810 iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 7801.6274iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.9490 iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 364.3334iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.8506 iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 16336.6465iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:33:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 12:34:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808069 38.839069 38.779335 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.667480 67.888672 67.593750 +util 96.500000 99.000000 93.000000 +[Apr 25, 2026 - 12:34:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 31 : avg iteration time 48.34 seconds +[Apr 25, 2026 - 12:34:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 31 : data loading time 0.00 +[Apr 25, 2026 - 12:34:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 31 : avg forward pass time 12.16 +[Apr 25, 2026 - 12:34:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 31 : backward pass time 11.53 +[Apr 25, 2026 - 12:34:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 31 : optimizer step time 0.96 +[Apr 25, 2026 - 12:34:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 31-------------------- +[Apr 25, 2026 - 12:34:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0129 iter count: 1.0 +[Apr 25, 2026 - 12:34:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[Apr 25, 2026 - 12:34:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:34:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:35:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808331 38.840054 38.779377 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.994629 68.003906 67.966797 +util 99.000000 100.000000 98.000000 +[Apr 25, 2026 - 12:35:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 32 : avg iteration time 49.95 seconds +[Apr 25, 2026 - 12:35:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 32 : data loading time 0.00 +[Apr 25, 2026 - 12:35:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 32 : avg forward pass time 12.94 +[Apr 25, 2026 - 12:35:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 32 : backward pass time 11.53 +[Apr 25, 2026 - 12:35:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 32 : optimizer step time 0.99 +[Apr 25, 2026 - 12:35:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 32-------------------- +[Apr 25, 2026 - 12:35:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 12:35:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 12:35:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:35:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:36:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808499 38.840794 38.779354 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.926270 67.986328 67.906250 +util 96.500000 100.000000 93.000000 +[Apr 25, 2026 - 12:36:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 33 : avg iteration time 47.13 seconds +[Apr 25, 2026 - 12:36:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 33 : data loading time 0.00 +[Apr 25, 2026 - 12:36:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 33 : avg forward pass time 11.53 +[Apr 25, 2026 - 12:36:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 33 : backward pass time 11.54 +[Apr 25, 2026 - 12:36:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 33 : optimizer step time 0.98 +[Apr 25, 2026 - 12:36:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 33-------------------- +[Apr 25, 2026 - 12:36:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0112 iter count: 1.0 +[Apr 25, 2026 - 12:36:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[Apr 25, 2026 - 12:36:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:36:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:37:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808503 38.840786 38.779377 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.911621 67.986328 67.886719 +util 96.250000 99.000000 93.000000 +[Apr 25, 2026 - 12:37:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 34 : avg iteration time 52.80 seconds +[Apr 25, 2026 - 12:37:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 34 : data loading time 0.00 +[Apr 25, 2026 - 12:37:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 34 : avg forward pass time 14.38 +[Apr 25, 2026 - 12:37:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 34 : backward pass time 11.54 +[Apr 25, 2026 - 12:37:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 34 : optimizer step time 0.98 +[Apr 25, 2026 - 12:37:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 34-------------------- +[Apr 25, 2026 - 12:37:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0093 iter count: 1.0 +[Apr 25, 2026 - 12:37:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[Apr 25, 2026 - 12:37:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:37:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808339 38.839516 38.779621 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.187012 133.210938 133.115234 +util 70.000000 78.000000 63.000000 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 35 : avg iteration time 172.16 seconds +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 35 : data loading time 0.00 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 35 : avg forward pass time 76.40 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 35 : backward pass time 9.14 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 35 : optimizer step time 0.32 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 35-------------------- +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 7066.0400 iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5187 iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 7066.0400iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.5227iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.2161 iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.0504 iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 11131.3164iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 11.3839iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 180014.7031iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5187iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.1098 iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 68.0084 iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 1.2161 iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 11.3839 iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.8823 iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 17406.2656iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.8083 iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 274.9346iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:40:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 12:40:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808416 38.839787 38.779621 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.560059 67.634766 67.535156 +util 96.750000 99.000000 94.000000 +[Apr 25, 2026 - 12:40:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 36 : avg iteration time 51.45 seconds +[Apr 25, 2026 - 12:40:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 36 : data loading time 0.00 +[Apr 25, 2026 - 12:40:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 36 : avg forward pass time 13.72 +[Apr 25, 2026 - 12:40:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 36 : backward pass time 11.51 +[Apr 25, 2026 - 12:40:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 36 : optimizer step time 0.97 +[Apr 25, 2026 - 12:40:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 36-------------------- +[Apr 25, 2026 - 12:40:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0123 iter count: 1.0 +[Apr 25, 2026 - 12:40:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[Apr 25, 2026 - 12:40:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:40:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:41:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808718 38.840988 38.779644 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.936035 67.984375 67.791016 +util 96.250000 99.000000 93.000000 +[Apr 25, 2026 - 12:41:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 37 : avg iteration time 46.91 seconds +[Apr 25, 2026 - 12:41:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 37 : data loading time 0.00 +[Apr 25, 2026 - 12:41:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 37 : avg forward pass time 11.45 +[Apr 25, 2026 - 12:41:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 37 : backward pass time 11.52 +[Apr 25, 2026 - 12:41:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 37 : optimizer step time 0.96 +[Apr 25, 2026 - 12:41:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 37-------------------- +[Apr 25, 2026 - 12:41:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0113 iter count: 1.0 +[Apr 25, 2026 - 12:41:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[Apr 25, 2026 - 12:41:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:41:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:42:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808779 38.841228 38.779644 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.936035 67.964844 67.849609 +util 97.250000 99.000000 92.000000 +[Apr 25, 2026 - 12:42:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 38 : avg iteration time 52.78 seconds +[Apr 25, 2026 - 12:42:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 38 : data loading time 0.00 +[Apr 25, 2026 - 12:42:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 38 : avg forward pass time 14.37 +[Apr 25, 2026 - 12:42:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 38 : backward pass time 11.51 +[Apr 25, 2026 - 12:42:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 38 : optimizer step time 0.99 +[Apr 25, 2026 - 12:42:36 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 38-------------------- +[Apr 25, 2026 - 12:42:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0108 iter count: 1.0 +[Apr 25, 2026 - 12:42:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 25, 2026 - 12:42:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:42:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:43:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808843 38.841499 38.779667 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.931152 67.964844 67.830078 +util 97.250000 99.000000 95.000000 +[Apr 25, 2026 - 12:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 39 : avg iteration time 49.84 seconds +[Apr 25, 2026 - 12:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 39 : data loading time 0.00 +[Apr 25, 2026 - 12:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 39 : avg forward pass time 12.89 +[Apr 25, 2026 - 12:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 39 : backward pass time 11.54 +[Apr 25, 2026 - 12:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 39 : optimizer step time 0.97 +[Apr 25, 2026 - 12:43:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 39-------------------- +[Apr 25, 2026 - 12:43:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0103 iter count: 1.0 +[Apr 25, 2026 - 12:43:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[Apr 25, 2026 - 12:43:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:43:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808387 38.839634 38.779667 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.114258 133.117188 133.113281 +util 70.750000 75.000000 66.000000 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 40 : avg iteration time 172.09 seconds +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 40 : data loading time 0.00 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 40 : avg forward pass time 76.20 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 40 : backward pass time 9.11 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 40 : optimizer step time 0.33 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 40-------------------- +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 8280.4499 iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6260 iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 8280.4499iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.7990iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5314 iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 5.9094 iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 12165.8262iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.8947 iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 135787.4062iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6260iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.2839 iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 711.8636iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.5269 iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 1157.2905iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.5819 iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 174.8334iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.9801 iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 2864.4363iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:46:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 12:47:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808500 38.840115 38.779659 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.975098 67.984375 67.947266 +util 97.750000 99.000000 96.000000 +[Apr 25, 2026 - 12:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 41 : avg iteration time 54.13 seconds +[Apr 25, 2026 - 12:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 41 : data loading time 0.00 +[Apr 25, 2026 - 12:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 41 : avg forward pass time 15.05 +[Apr 25, 2026 - 12:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 41 : backward pass time 11.53 +[Apr 25, 2026 - 12:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 41 : optimizer step time 0.98 +[Apr 25, 2026 - 12:47:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 41-------------------- +[Apr 25, 2026 - 12:47:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0106 iter count: 1.0 +[Apr 25, 2026 - 12:47:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[Apr 25, 2026 - 12:47:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:47:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:48:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808800 38.841339 38.779633 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.931152 67.964844 67.830078 +util 96.250000 99.000000 91.000000 +[Apr 25, 2026 - 12:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 42 : avg iteration time 49.81 seconds +[Apr 25, 2026 - 12:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 42 : data loading time 0.00 +[Apr 25, 2026 - 12:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 42 : avg forward pass time 12.89 +[Apr 25, 2026 - 12:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 42 : backward pass time 11.52 +[Apr 25, 2026 - 12:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 42 : optimizer step time 0.97 +[Apr 25, 2026 - 12:48:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 42-------------------- +[Apr 25, 2026 - 12:48:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0116 iter count: 1.0 +[Apr 25, 2026 - 12:48:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[Apr 25, 2026 - 12:48:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:48:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:48:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808868 38.841587 38.779644 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.911621 67.945312 67.810547 +util 96.250000 99.000000 91.000000 +[Apr 25, 2026 - 12:48:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 43 : avg iteration time 47.00 seconds +[Apr 25, 2026 - 12:48:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 43 : data loading time 0.00 +[Apr 25, 2026 - 12:48:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 43 : avg forward pass time 11.48 +[Apr 25, 2026 - 12:48:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 43 : backward pass time 11.53 +[Apr 25, 2026 - 12:48:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 43 : optimizer step time 0.96 +[Apr 25, 2026 - 12:48:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 43-------------------- +[Apr 25, 2026 - 12:48:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 25, 2026 - 12:48:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 25, 2026 - 12:48:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:48:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:49:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808703 38.840935 38.779640 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.950684 67.984375 67.849609 +util 98.750000 100.000000 97.000000 +[Apr 25, 2026 - 12:49:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 44 : avg iteration time 49.88 seconds +[Apr 25, 2026 - 12:49:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 44 : data loading time 0.00 +[Apr 25, 2026 - 12:49:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 44 : avg forward pass time 12.91 +[Apr 25, 2026 - 12:49:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 44 : backward pass time 11.53 +[Apr 25, 2026 - 12:49:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 44 : optimizer step time 0.97 +[Apr 25, 2026 - 12:49:39 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 44-------------------- +[Apr 25, 2026 - 12:49:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0107 iter count: 1.0 +[Apr 25, 2026 - 12:49:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[Apr 25, 2026 - 12:49:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:49:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808541 38.840248 38.779629 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.099121 133.115234 133.093750 +util 72.750000 78.000000 67.000000 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 45 : avg iteration time 172.50 seconds +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 45 : data loading time 0.00 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 45 : avg forward pass time 76.99 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 45 : backward pass time 9.11 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 45 : optimizer step time 0.32 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 45-------------------- +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 54945.4487 iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6718 iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 54945.4487iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4873iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.2638 iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0498 iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 101963.0703iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 92.5353iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1328599.7500iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6718iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.6841 iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 639284.5625iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.1906 iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 590.6873iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.0728 iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 25478.2969iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.2084 iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 33415.8047iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:52:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 12:53:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808591 38.840469 38.779629 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.857910 67.925781 67.654297 +util 96.250000 99.000000 93.000000 +[Apr 25, 2026 - 12:53:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 46 : avg iteration time 54.37 seconds +[Apr 25, 2026 - 12:53:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 46 : data loading time 0.00 +[Apr 25, 2026 - 12:53:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 46 : avg forward pass time 15.16 +[Apr 25, 2026 - 12:53:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 46 : backward pass time 11.54 +[Apr 25, 2026 - 12:53:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 46 : optimizer step time 0.97 +[Apr 25, 2026 - 12:53:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 46-------------------- +[Apr 25, 2026 - 12:53:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0106 iter count: 1.0 +[Apr 25, 2026 - 12:53:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[Apr 25, 2026 - 12:53:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:53:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:54:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809052 38.842278 38.779629 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.584473 67.673828 67.554688 +util 96.500000 100.000000 93.000000 +[Apr 25, 2026 - 12:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 47 : avg iteration time 49.80 seconds +[Apr 25, 2026 - 12:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 47 : data loading time 0.00 +[Apr 25, 2026 - 12:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 47 : avg forward pass time 12.89 +[Apr 25, 2026 - 12:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 47 : backward pass time 11.53 +[Apr 25, 2026 - 12:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 47 : optimizer step time 0.98 +[Apr 25, 2026 - 12:54:15 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 47-------------------- +[Apr 25, 2026 - 12:54:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 12:54:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 12:54:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:54:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:55:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809052 38.842289 38.779629 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.960449 67.964844 67.947266 +util 96.250000 99.000000 93.000000 +[Apr 25, 2026 - 12:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 48 : avg iteration time 52.88 seconds +[Apr 25, 2026 - 12:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 48 : data loading time 0.00 +[Apr 25, 2026 - 12:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 48 : avg forward pass time 14.42 +[Apr 25, 2026 - 12:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 48 : backward pass time 11.53 +[Apr 25, 2026 - 12:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 48 : optimizer step time 0.97 +[Apr 25, 2026 - 12:55:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 48-------------------- +[Apr 25, 2026 - 12:55:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0117 iter count: 1.0 +[Apr 25, 2026 - 12:55:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[Apr 25, 2026 - 12:55:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:55:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:56:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808992 38.842026 38.779636 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.975098 67.984375 67.947266 +util 97.250000 99.000000 95.000000 +[Apr 25, 2026 - 12:56:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 49 : avg iteration time 52.75 seconds +[Apr 25, 2026 - 12:56:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 49 : data loading time 0.00 +[Apr 25, 2026 - 12:56:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 49 : avg forward pass time 14.34 +[Apr 25, 2026 - 12:56:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 49 : backward pass time 11.52 +[Apr 25, 2026 - 12:56:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 49 : optimizer step time 0.99 +[Apr 25, 2026 - 12:56:01 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 49-------------------- +[Apr 25, 2026 - 12:56:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0109 iter count: 1.0 +[Apr 25, 2026 - 12:56:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[Apr 25, 2026 - 12:56:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:56:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808723 38.840973 38.779640 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.094238 133.113281 133.037109 +util 67.750000 71.000000 63.000000 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 50 : avg iteration time 169.23 seconds +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 50 : data loading time 0.00 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 50 : avg forward pass time 74.76 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 50 : backward pass time 9.14 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 50 : optimizer step time 0.32 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 50-------------------- +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 68330.8356 iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6999 iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 68330.8356iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3686iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.7007 iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.3304 iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 128943.5469iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 30.0034iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2328578.5000iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6999iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.3867 iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 6459.6519iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.1798 iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 4270.7988iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.1074 iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 67.6835 iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.0287 iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 3156.7693iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:58:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 12:59:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808722 38.840973 38.779636 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.853027 67.925781 67.634766 +util 95.750000 99.000000 92.000000 +[Apr 25, 2026 - 12:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 51 : avg iteration time 51.22 seconds +[Apr 25, 2026 - 12:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 51 : data loading time 0.00 +[Apr 25, 2026 - 12:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 51 : avg forward pass time 13.59 +[Apr 25, 2026 - 12:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 51 : backward pass time 11.53 +[Apr 25, 2026 - 12:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 51 : optimizer step time 0.97 +[Apr 25, 2026 - 12:59:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 51-------------------- +[Apr 25, 2026 - 12:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0103 iter count: 1.0 +[Apr 25, 2026 - 12:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[Apr 25, 2026 - 12:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 12:59:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:00:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808734 38.840973 38.779636 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.028809 68.103516 68.003906 +util 95.000000 99.000000 91.000000 +[Apr 25, 2026 - 13:00:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 52 : avg iteration time 49.79 seconds +[Apr 25, 2026 - 13:00:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 52 : data loading time 0.00 +[Apr 25, 2026 - 13:00:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 52 : avg forward pass time 12.87 +[Apr 25, 2026 - 13:00:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 52 : backward pass time 11.53 +[Apr 25, 2026 - 13:00:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 52 : optimizer step time 0.97 +[Apr 25, 2026 - 13:00:31 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 52-------------------- +[Apr 25, 2026 - 13:00:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0122 iter count: 1.0 +[Apr 25, 2026 - 13:00:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[Apr 25, 2026 - 13:00:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:00:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:01:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808728 38.840981 38.779636 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.882324 67.964844 67.634766 +util 95.500000 100.000000 91.000000 +[Apr 25, 2026 - 13:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 53 : avg iteration time 49.92 seconds +[Apr 25, 2026 - 13:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 53 : data loading time 0.00 +[Apr 25, 2026 - 13:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 53 : avg forward pass time 12.93 +[Apr 25, 2026 - 13:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 53 : backward pass time 11.55 +[Apr 25, 2026 - 13:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 53 : optimizer step time 0.96 +[Apr 25, 2026 - 13:01:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 53-------------------- +[Apr 25, 2026 - 13:01:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 13:01:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 13:01:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:01:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:02:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808790 38.841221 38.779636 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.887207 67.964844 67.654297 +util 97.500000 100.000000 92.000000 +[Apr 25, 2026 - 13:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 54 : avg iteration time 49.80 seconds +[Apr 25, 2026 - 13:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 54 : data loading time 0.00 +[Apr 25, 2026 - 13:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 54 : avg forward pass time 12.89 +[Apr 25, 2026 - 13:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 54 : backward pass time 11.52 +[Apr 25, 2026 - 13:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 54 : optimizer step time 0.98 +[Apr 25, 2026 - 13:02:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 54-------------------- +[Apr 25, 2026 - 13:02:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0118 iter count: 1.0 +[Apr 25, 2026 - 13:02:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[Apr 25, 2026 - 13:02:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:02:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808674 38.840733 38.779636 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.143555 133.152344 133.117188 +util 72.250000 78.000000 68.000000 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 55 : avg iteration time 172.59 seconds +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 55 : data loading time 0.00 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 55 : avg forward pass time 77.03 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 55 : backward pass time 9.11 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 55 : optimizer step time 0.32 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 55-------------------- +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 18282.0396 iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6124 iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 18282.0396iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.7511iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.0130 iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.3827 iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 33365.9688iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 7.5838 iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 349894.6875iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6124iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.6273 iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 10453.4199iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.0178 iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 56.5771 iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.2265 iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 634.6547iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.9227 iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 345.6071iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:05:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 13:05:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808727 38.840950 38.779659 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.750488 67.789062 67.634766 +util 97.000000 99.000000 91.000000 +[Apr 25, 2026 - 13:05:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 56 : avg iteration time 48.20 seconds +[Apr 25, 2026 - 13:05:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 56 : data loading time 0.00 +[Apr 25, 2026 - 13:05:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 56 : avg forward pass time 12.09 +[Apr 25, 2026 - 13:05:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 56 : backward pass time 11.52 +[Apr 25, 2026 - 13:05:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 56 : optimizer step time 0.97 +[Apr 25, 2026 - 13:05:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 56-------------------- +[Apr 25, 2026 - 13:05:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0117 iter count: 1.0 +[Apr 25, 2026 - 13:05:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[Apr 25, 2026 - 13:05:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:05:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:06:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808721 38.840942 38.779636 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.774902 67.828125 67.615234 +util 96.750000 99.000000 94.000000 +[Apr 25, 2026 - 13:06:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 57 : avg iteration time 52.85 seconds +[Apr 25, 2026 - 13:06:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 57 : data loading time 0.00 +[Apr 25, 2026 - 13:06:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 57 : avg forward pass time 14.41 +[Apr 25, 2026 - 13:06:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 57 : backward pass time 11.53 +[Apr 25, 2026 - 13:06:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 57 : optimizer step time 0.96 +[Apr 25, 2026 - 13:06:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 57-------------------- +[Apr 25, 2026 - 13:06:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0118 iter count: 1.0 +[Apr 25, 2026 - 13:06:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[Apr 25, 2026 - 13:06:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:06:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:07:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808732 38.840965 38.779636 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.667480 67.712891 67.652344 +util 96.750000 99.000000 91.000000 +[Apr 25, 2026 - 13:07:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 58 : avg iteration time 49.84 seconds +[Apr 25, 2026 - 13:07:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 58 : data loading time 0.00 +[Apr 25, 2026 - 13:07:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 58 : avg forward pass time 12.90 +[Apr 25, 2026 - 13:07:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 58 : backward pass time 11.53 +[Apr 25, 2026 - 13:07:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 58 : optimizer step time 0.98 +[Apr 25, 2026 - 13:07:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 58-------------------- +[Apr 25, 2026 - 13:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0110 iter count: 1.0 +[Apr 25, 2026 - 13:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[Apr 25, 2026 - 13:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:07:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:08:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808732 38.840988 38.779636 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.770020 67.828125 67.595703 +util 95.250000 99.000000 91.000000 +[Apr 25, 2026 - 13:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 59 : avg iteration time 49.81 seconds +[Apr 25, 2026 - 13:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 59 : data loading time 0.00 +[Apr 25, 2026 - 13:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 59 : avg forward pass time 12.89 +[Apr 25, 2026 - 13:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 59 : backward pass time 11.52 +[Apr 25, 2026 - 13:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 59 : optimizer step time 0.98 +[Apr 25, 2026 - 13:08:24 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 59-------------------- +[Apr 25, 2026 - 13:08:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 13:08:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 13:08:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:08:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808739 38.840988 38.779636 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.128418 133.132812 133.115234 +util 70.250000 75.000000 65.000000 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 60 : avg iteration time 172.13 seconds +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 60 : data loading time 0.00 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 60 : avg forward pass time 76.40 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 60 : backward pass time 9.12 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 60 : optimizer step time 0.32 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 60-------------------- +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 16323.3220 iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5729 iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 16323.3220iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.8464iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.7627 iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.1834 iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 27898.2773iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 33.9682iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 234851.9844iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5729iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.6158 iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1382.3613iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.2752 iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 5168.5986iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 1.9391 iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 48.3380 iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.4857 iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1065.7313iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:11:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 13:12:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808736 38.840942 38.779636 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.058105 68.062500 68.044922 +util 97.500000 100.000000 95.000000 +[Apr 25, 2026 - 13:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 61 : avg iteration time 54.11 seconds +[Apr 25, 2026 - 13:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 61 : data loading time 0.00 +[Apr 25, 2026 - 13:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 61 : avg forward pass time 15.03 +[Apr 25, 2026 - 13:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 61 : backward pass time 11.54 +[Apr 25, 2026 - 13:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 61 : optimizer step time 0.97 +[Apr 25, 2026 - 13:12:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 61-------------------- +[Apr 25, 2026 - 13:12:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0139 iter count: 1.0 +[Apr 25, 2026 - 13:12:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[Apr 25, 2026 - 13:12:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:12:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:13:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808805 38.841225 38.779636 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.111816 68.160156 67.966797 +util 97.250000 99.000000 95.000000 +[Apr 25, 2026 - 13:13:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 62 : avg iteration time 49.91 seconds +[Apr 25, 2026 - 13:13:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 62 : data loading time 0.00 +[Apr 25, 2026 - 13:13:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 62 : avg forward pass time 12.93 +[Apr 25, 2026 - 13:13:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 62 : backward pass time 11.54 +[Apr 25, 2026 - 13:13:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 62 : optimizer step time 0.98 +[Apr 25, 2026 - 13:13:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 62-------------------- +[Apr 25, 2026 - 13:13:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0134 iter count: 1.0 +[Apr 25, 2026 - 13:13:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[Apr 25, 2026 - 13:13:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:13:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:13:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808809 38.841244 38.779659 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.058105 68.082031 67.986328 +util 98.500000 100.000000 97.000000 +[Apr 25, 2026 - 13:13:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 63 : avg iteration time 46.91 seconds +[Apr 25, 2026 - 13:13:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 63 : data loading time 0.00 +[Apr 25, 2026 - 13:13:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 63 : avg forward pass time 11.44 +[Apr 25, 2026 - 13:13:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 63 : backward pass time 11.51 +[Apr 25, 2026 - 13:13:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 63 : optimizer step time 0.99 +[Apr 25, 2026 - 13:13:47 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 63-------------------- +[Apr 25, 2026 - 13:13:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0119 iter count: 1.0 +[Apr 25, 2026 - 13:13:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 25, 2026 - 13:13:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:13:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:14:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808815 38.841236 38.779644 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.136230 68.179688 68.005859 +util 95.500000 99.000000 92.000000 +[Apr 25, 2026 - 13:14:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 64 : avg iteration time 49.94 seconds +[Apr 25, 2026 - 13:14:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 64 : data loading time 0.00 +[Apr 25, 2026 - 13:14:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 64 : avg forward pass time 12.95 +[Apr 25, 2026 - 13:14:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 64 : backward pass time 11.53 +[Apr 25, 2026 - 13:14:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 64 : optimizer step time 0.97 +[Apr 25, 2026 - 13:14:37 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 64-------------------- +[Apr 25, 2026 - 13:14:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0123 iter count: 1.0 +[Apr 25, 2026 - 13:14:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[Apr 25, 2026 - 13:14:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:14:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808824 38.841232 38.779682 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.138184 133.154297 133.132812 +util 68.500000 73.000000 64.000000 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 65 : avg iteration time 174.92 seconds +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 65 : data loading time 0.00 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 65 : avg forward pass time 77.62 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 65 : backward pass time 9.13 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 65 : optimizer step time 0.32 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 65-------------------- +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 58382.3777 iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6320 iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 58382.3777iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.9624iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7461 iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0736 iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 96065.4375iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.4469 iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1393229.2500iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6320iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.3487 iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 810.3668iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.3932 iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 6544.6655iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.3505 iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 813.1521iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.6370 iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1442.3497iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 13:18:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808810 38.841213 38.779682 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.867676 67.945312 67.634766 +util 95.500000 99.000000 92.000000 +[Apr 25, 2026 - 13:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 66 : avg iteration time 51.28 seconds +[Apr 25, 2026 - 13:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 66 : data loading time 0.00 +[Apr 25, 2026 - 13:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 66 : avg forward pass time 13.62 +[Apr 25, 2026 - 13:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 66 : backward pass time 11.53 +[Apr 25, 2026 - 13:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 66 : optimizer step time 0.99 +[Apr 25, 2026 - 13:18:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 66-------------------- +[Apr 25, 2026 - 13:18:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0138 iter count: 1.0 +[Apr 25, 2026 - 13:18:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0138 iter count: 1.0 +[Apr 25, 2026 - 13:18:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:18:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:19:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808818 38.841213 38.779682 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.779785 67.828125 67.634766 +util 98.250000 99.000000 97.000000 +[Apr 25, 2026 - 13:19:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 67 : avg iteration time 49.87 seconds +[Apr 25, 2026 - 13:19:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 67 : data loading time 0.00 +[Apr 25, 2026 - 13:19:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 67 : avg forward pass time 12.92 +[Apr 25, 2026 - 13:19:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 67 : backward pass time 11.55 +[Apr 25, 2026 - 13:19:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 67 : optimizer step time 0.96 +[Apr 25, 2026 - 13:19:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 67-------------------- +[Apr 25, 2026 - 13:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 13:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 13:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:19:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:20:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808799 38.841213 38.779659 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.916504 67.964844 67.771484 +util 97.250000 99.000000 95.000000 +[Apr 25, 2026 - 13:20:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 68 : avg iteration time 52.69 seconds +[Apr 25, 2026 - 13:20:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 68 : data loading time 0.00 +[Apr 25, 2026 - 13:20:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 68 : avg forward pass time 14.34 +[Apr 25, 2026 - 13:20:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 68 : backward pass time 11.53 +[Apr 25, 2026 - 13:20:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 68 : optimizer step time 0.97 +[Apr 25, 2026 - 13:20:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 68-------------------- +[Apr 25, 2026 - 13:20:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0124 iter count: 1.0 +[Apr 25, 2026 - 13:20:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[Apr 25, 2026 - 13:20:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:20:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:20:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808805 38.841213 38.779659 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.979980 68.025391 67.964844 +util 95.250000 99.000000 91.000000 +[Apr 25, 2026 - 13:20:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 69 : avg iteration time 52.75 seconds +[Apr 25, 2026 - 13:20:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 69 : data loading time 0.00 +[Apr 25, 2026 - 13:20:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 69 : avg forward pass time 14.37 +[Apr 25, 2026 - 13:20:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 69 : backward pass time 11.53 +[Apr 25, 2026 - 13:20:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 69 : optimizer step time 0.97 +[Apr 25, 2026 - 13:20:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 69-------------------- +[Apr 25, 2026 - 13:20:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 25, 2026 - 13:20:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 25, 2026 - 13:20:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:20:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808503 38.840046 38.779636 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.118652 133.132812 133.076172 +util 69.000000 73.000000 65.000000 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 70 : avg iteration time 171.91 seconds +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 70 : data loading time 0.00 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 70 : avg forward pass time 76.14 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 70 : backward pass time 9.10 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 70 : optimizer step time 0.32 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 70-------------------- +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 44124.5889 iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6323 iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 44124.5889iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.0169iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.9840 iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.1664 iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 84688.7031iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 7.1562 iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1677373.7500iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6323iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.8864 iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 321.4618iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.8760 iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2326.3330iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 1.6333 iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 26.2214 iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.9632 iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 2769.6692iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:23:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 13:24:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808517 38.840031 38.779644 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.789551 67.847656 67.615234 +util 98.250000 100.000000 97.000000 +[Apr 25, 2026 - 13:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 71 : avg iteration time 51.24 seconds +[Apr 25, 2026 - 13:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 71 : data loading time 0.00 +[Apr 25, 2026 - 13:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 71 : avg forward pass time 13.61 +[Apr 25, 2026 - 13:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 71 : backward pass time 11.51 +[Apr 25, 2026 - 13:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 71 : optimizer step time 0.98 +[Apr 25, 2026 - 13:24:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 71-------------------- +[Apr 25, 2026 - 13:24:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0143 iter count: 1.0 +[Apr 25, 2026 - 13:24:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0143 iter count: 1.0 +[Apr 25, 2026 - 13:24:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:24:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:25:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808875 38.841530 38.779636 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.691895 67.810547 67.652344 +util 97.500000 99.000000 96.000000 +[Apr 25, 2026 - 13:25:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 72 : avg iteration time 49.79 seconds +[Apr 25, 2026 - 13:25:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 72 : data loading time 0.00 +[Apr 25, 2026 - 13:25:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 72 : avg forward pass time 12.88 +[Apr 25, 2026 - 13:25:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 72 : backward pass time 11.52 +[Apr 25, 2026 - 13:25:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 72 : optimizer step time 0.97 +[Apr 25, 2026 - 13:25:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 72-------------------- +[Apr 25, 2026 - 13:25:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0130 iter count: 1.0 +[Apr 25, 2026 - 13:25:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[Apr 25, 2026 - 13:25:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:25:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:26:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809127 38.842518 38.779636 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.740723 67.789062 67.595703 +util 96.750000 99.000000 94.000000 +[Apr 25, 2026 - 13:26:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 73 : avg iteration time 52.78 seconds +[Apr 25, 2026 - 13:26:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 73 : data loading time 0.00 +[Apr 25, 2026 - 13:26:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 73 : avg forward pass time 14.37 +[Apr 25, 2026 - 13:26:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 73 : backward pass time 11.54 +[Apr 25, 2026 - 13:26:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 73 : optimizer step time 0.97 +[Apr 25, 2026 - 13:26:24 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 73-------------------- +[Apr 25, 2026 - 13:26:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0120 iter count: 1.0 +[Apr 25, 2026 - 13:26:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[Apr 25, 2026 - 13:26:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:26:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:27:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809127 38.842518 38.779636 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.843262 67.847656 67.830078 +util 98.250000 99.000000 97.000000 +[Apr 25, 2026 - 13:27:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 74 : avg iteration time 52.75 seconds +[Apr 25, 2026 - 13:27:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 74 : data loading time 0.00 +[Apr 25, 2026 - 13:27:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 74 : avg forward pass time 14.35 +[Apr 25, 2026 - 13:27:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 74 : backward pass time 11.54 +[Apr 25, 2026 - 13:27:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 74 : optimizer step time 0.97 +[Apr 25, 2026 - 13:27:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 74-------------------- +[Apr 25, 2026 - 13:27:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0130 iter count: 1.0 +[Apr 25, 2026 - 13:27:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[Apr 25, 2026 - 13:27:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:27:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808703 38.840328 38.779743 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.079590 133.095703 133.074219 +util 71.000000 75.000000 67.000000 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 75 : avg iteration time 172.26 seconds +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 75 : data loading time 0.00 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 75 : avg forward pass time 76.39 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 75 : backward pass time 9.16 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 75 : optimizer step time 0.32 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 75-------------------- +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 99269.9414 iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4707 iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 99269.9414iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3143iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.1897 iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.4672 iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 227712.6406iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 10.7980iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 3061384.0000iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4707iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.9253 iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 18968.5449iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.6330 iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 1430.8495iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.6019 iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 181.9558iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.3219 iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 5675.0796iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:30:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 13:31:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808824 38.840832 38.779743 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.940918 67.984375 67.810547 +util 96.000000 99.000000 93.000000 +[Apr 25, 2026 - 13:31:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 76 : avg iteration time 54.17 seconds +[Apr 25, 2026 - 13:31:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 76 : data loading time 0.00 +[Apr 25, 2026 - 13:31:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 76 : avg forward pass time 15.08 +[Apr 25, 2026 - 13:31:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 76 : backward pass time 11.53 +[Apr 25, 2026 - 13:31:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 76 : optimizer step time 0.98 +[Apr 25, 2026 - 13:31:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 76-------------------- +[Apr 25, 2026 - 13:31:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0131 iter count: 1.0 +[Apr 25, 2026 - 13:31:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0131 iter count: 1.0 +[Apr 25, 2026 - 13:31:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:31:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:31:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809134 38.842072 38.779743 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.940918 67.984375 67.810547 +util 98.000000 100.000000 96.000000 +[Apr 25, 2026 - 13:31:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 77 : avg iteration time 52.64 seconds +[Apr 25, 2026 - 13:31:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 77 : data loading time 0.00 +[Apr 25, 2026 - 13:31:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 77 : avg forward pass time 14.31 +[Apr 25, 2026 - 13:31:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 77 : backward pass time 11.50 +[Apr 25, 2026 - 13:31:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 77 : optimizer step time 0.97 +[Apr 25, 2026 - 13:31:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 77-------------------- +[Apr 25, 2026 - 13:31:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0141 iter count: 1.0 +[Apr 25, 2026 - 13:31:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0141 iter count: 1.0 +[Apr 25, 2026 - 13:31:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:31:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:32:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809430 38.843258 38.779743 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.901855 67.945312 67.771484 +util 98.250000 99.000000 97.000000 +[Apr 25, 2026 - 13:32:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 78 : avg iteration time 52.73 seconds +[Apr 25, 2026 - 13:32:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 78 : data loading time 0.00 +[Apr 25, 2026 - 13:32:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 78 : avg forward pass time 14.35 +[Apr 25, 2026 - 13:32:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 78 : backward pass time 11.52 +[Apr 25, 2026 - 13:32:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 78 : optimizer step time 0.99 +[Apr 25, 2026 - 13:32:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 78-------------------- +[Apr 25, 2026 - 13:32:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0127 iter count: 1.0 +[Apr 25, 2026 - 13:32:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[Apr 25, 2026 - 13:32:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:32:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:33:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809274 38.842621 38.779755 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.911621 68.003906 67.634766 +util 96.750000 99.000000 94.000000 +[Apr 25, 2026 - 13:33:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 79 : avg iteration time 52.66 seconds +[Apr 25, 2026 - 13:33:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 79 : data loading time 0.00 +[Apr 25, 2026 - 13:33:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 79 : avg forward pass time 14.32 +[Apr 25, 2026 - 13:33:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 79 : backward pass time 11.52 +[Apr 25, 2026 - 13:33:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 79 : optimizer step time 0.98 +[Apr 25, 2026 - 13:33:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 79-------------------- +[Apr 25, 2026 - 13:33:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0147 iter count: 1.0 +[Apr 25, 2026 - 13:33:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0147 iter count: 1.0 +[Apr 25, 2026 - 13:33:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:33:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809887 38.842537 38.780609 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.149414 133.154297 133.134766 +util 69.250000 73.000000 65.000000 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 80 : avg iteration time 166.79 seconds +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 80 : data loading time 0.00 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 80 : avg forward pass time 74.14 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 80 : backward pass time 9.12 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 80 : optimizer step time 0.32 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 80-------------------- +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 105635.3099 iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5738 iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 105635.3099iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.6784iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.9269 iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.2882 iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 148180.2031iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 6.3844 iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2139988.7500iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5738iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 0.9849 iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 7.1686 iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 7.2882 iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2139988.7500iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.8754 iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 17169.4336iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.2074 iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 82.6635 iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:36:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 13:37:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809884 38.842537 38.780602 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.892090 67.984375 67.615234 +util 96.750000 99.000000 94.000000 +[Apr 25, 2026 - 13:37:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 81 : avg iteration time 48.32 seconds +[Apr 25, 2026 - 13:37:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 81 : data loading time 0.00 +[Apr 25, 2026 - 13:37:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 81 : avg forward pass time 12.14 +[Apr 25, 2026 - 13:37:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 81 : backward pass time 11.52 +[Apr 25, 2026 - 13:37:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 81 : optimizer step time 0.99 +[Apr 25, 2026 - 13:37:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 81-------------------- +[Apr 25, 2026 - 13:37:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0146 iter count: 1.0 +[Apr 25, 2026 - 13:37:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0146 iter count: 1.0 +[Apr 25, 2026 - 13:37:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:37:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:38:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809896 38.842560 38.780624 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.931152 67.964844 67.830078 +util 95.500000 99.000000 92.000000 +[Apr 25, 2026 - 13:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 82 : avg iteration time 46.87 seconds +[Apr 25, 2026 - 13:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 82 : data loading time 0.00 +[Apr 25, 2026 - 13:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 82 : avg forward pass time 11.42 +[Apr 25, 2026 - 13:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 82 : backward pass time 11.52 +[Apr 25, 2026 - 13:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 82 : optimizer step time 0.97 +[Apr 25, 2026 - 13:38:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 82-------------------- +[Apr 25, 2026 - 13:38:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0145 iter count: 1.0 +[Apr 25, 2026 - 13:38:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0145 iter count: 1.0 +[Apr 25, 2026 - 13:38:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:38:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:38:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809944 38.842789 38.780605 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.862793 67.886719 67.791016 +util 98.000000 99.000000 97.000000 +[Apr 25, 2026 - 13:38:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 83 : avg iteration time 49.85 seconds +[Apr 25, 2026 - 13:38:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 83 : data loading time 0.00 +[Apr 25, 2026 - 13:38:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 83 : avg forward pass time 12.90 +[Apr 25, 2026 - 13:38:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 83 : backward pass time 11.52 +[Apr 25, 2026 - 13:38:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 83 : optimizer step time 0.98 +[Apr 25, 2026 - 13:38:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 83-------------------- +[Apr 25, 2026 - 13:38:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0128 iter count: 1.0 +[Apr 25, 2026 - 13:38:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[Apr 25, 2026 - 13:38:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:38:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:39:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810005 38.843040 38.780602 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.872559 67.964844 67.595703 +util 99.250000 100.000000 99.000000 +[Apr 25, 2026 - 13:39:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 84 : avg iteration time 49.82 seconds +[Apr 25, 2026 - 13:39:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 84 : data loading time 0.00 +[Apr 25, 2026 - 13:39:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 84 : avg forward pass time 12.90 +[Apr 25, 2026 - 13:39:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 84 : backward pass time 11.51 +[Apr 25, 2026 - 13:39:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 84 : optimizer step time 0.98 +[Apr 25, 2026 - 13:39:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 84-------------------- +[Apr 25, 2026 - 13:39:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0146 iter count: 1.0 +[Apr 25, 2026 - 13:39:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0146 iter count: 1.0 +[Apr 25, 2026 - 13:39:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:39:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809954 38.842724 38.780602 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.157715 133.171875 133.115234 +util 68.000000 72.000000 64.000000 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 85 : avg iteration time 172.12 seconds +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 85 : data loading time 0.00 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 85 : avg forward pass time 76.34 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 85 : backward pass time 9.16 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 85 : optimizer step time 0.32 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 85-------------------- +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 6921.9611 iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5729 iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 6921.9611iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.7335iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.6436 iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 5.9044 iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 14687.7676iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 26.7663iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 134429.8594iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5729iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.5355 iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 159.3441iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.4463 iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 133.3125iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.5953 iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 179.5786iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.9044 iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 134429.8594iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0155 iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:42:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 13:43:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810084 38.843224 38.780624 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.955566 67.986328 67.945312 +util 96.750000 99.000000 94.000000 +[Apr 25, 2026 - 13:43:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 86 : avg iteration time 51.23 seconds +[Apr 25, 2026 - 13:43:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 86 : data loading time 0.00 +[Apr 25, 2026 - 13:43:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 86 : avg forward pass time 13.61 +[Apr 25, 2026 - 13:43:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 86 : backward pass time 11.53 +[Apr 25, 2026 - 13:43:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 86 : optimizer step time 0.97 +[Apr 25, 2026 - 13:43:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 86-------------------- +[Apr 25, 2026 - 13:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0138 iter count: 1.0 +[Apr 25, 2026 - 13:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0138 iter count: 1.0 +[Apr 25, 2026 - 13:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:43:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:44:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810081 38.843224 38.780602 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.896973 67.986328 67.867188 +util 98.250000 100.000000 97.000000 +[Apr 25, 2026 - 13:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 87 : avg iteration time 49.83 seconds +[Apr 25, 2026 - 13:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 87 : data loading time 0.00 +[Apr 25, 2026 - 13:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 87 : avg forward pass time 12.90 +[Apr 25, 2026 - 13:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 87 : backward pass time 11.53 +[Apr 25, 2026 - 13:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 87 : optimizer step time 0.97 +[Apr 25, 2026 - 13:44:16 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 87-------------------- +[Apr 25, 2026 - 13:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0148 iter count: 1.0 +[Apr 25, 2026 - 13:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0148 iter count: 1.0 +[Apr 25, 2026 - 13:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:44:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:45:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810102 38.843269 38.780602 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.750488 67.828125 67.517578 +util 96.500000 100.000000 93.000000 +[Apr 25, 2026 - 13:45:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 88 : avg iteration time 49.88 seconds +[Apr 25, 2026 - 13:45:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 88 : data loading time 0.00 +[Apr 25, 2026 - 13:45:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 88 : avg forward pass time 12.91 +[Apr 25, 2026 - 13:45:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 88 : backward pass time 11.56 +[Apr 25, 2026 - 13:45:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 88 : optimizer step time 0.97 +[Apr 25, 2026 - 13:45:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 88-------------------- +[Apr 25, 2026 - 13:45:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0150 iter count: 1.0 +[Apr 25, 2026 - 13:45:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0150 iter count: 1.0 +[Apr 25, 2026 - 13:45:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:45:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:45:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810221 38.843769 38.780602 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.828613 67.847656 67.771484 +util 97.000000 99.000000 95.000000 +[Apr 25, 2026 - 13:45:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 89 : avg iteration time 49.82 seconds +[Apr 25, 2026 - 13:45:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 89 : data loading time 0.00 +[Apr 25, 2026 - 13:45:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 89 : avg forward pass time 12.88 +[Apr 25, 2026 - 13:45:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 89 : backward pass time 11.52 +[Apr 25, 2026 - 13:45:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 89 : optimizer step time 0.99 +[Apr 25, 2026 - 13:45:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 89-------------------- +[Apr 25, 2026 - 13:45:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0152 iter count: 1.0 +[Apr 25, 2026 - 13:45:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0152 iter count: 1.0 +[Apr 25, 2026 - 13:45:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:45:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810140 38.843273 38.780663 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.135254 133.154297 133.078125 +util 71.500000 76.000000 67.000000 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 90 : avg iteration time 175.20 seconds +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 90 : data loading time 0.00 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 90 : avg forward pass time 78.35 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 90 : backward pass time 9.12 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 90 : optimizer step time 0.32 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 90-------------------- +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 23923.5348 iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6296 iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 23923.5348iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.7124iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.8768 iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.9545 iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 50928.3516iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 5.7757 iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1097889.5000iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6296iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.9821 iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 389.2798iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.6675 iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 83695.4609iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.9413 iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 358.7410iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.0280 iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 426.6457iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0138 iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:48:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 13:49:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810267 38.843784 38.780663 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.872559 67.888672 67.867188 +util 95.500000 99.000000 92.000000 +[Apr 25, 2026 - 13:49:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 91 : avg iteration time 48.46 seconds +[Apr 25, 2026 - 13:49:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 91 : data loading time 0.00 +[Apr 25, 2026 - 13:49:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 91 : avg forward pass time 12.20 +[Apr 25, 2026 - 13:49:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 91 : backward pass time 11.52 +[Apr 25, 2026 - 13:49:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 91 : optimizer step time 0.98 +[Apr 25, 2026 - 13:49:40 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 91-------------------- +[Apr 25, 2026 - 13:49:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0162 iter count: 1.0 +[Apr 25, 2026 - 13:49:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0162 iter count: 1.0 +[Apr 25, 2026 - 13:49:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:49:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:50:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810264 38.843746 38.780685 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.672363 67.849609 67.613281 +util 96.500000 100.000000 93.000000 +[Apr 25, 2026 - 13:50:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 92 : avg iteration time 52.72 seconds +[Apr 25, 2026 - 13:50:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 92 : data loading time 0.00 +[Apr 25, 2026 - 13:50:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 92 : avg forward pass time 14.34 +[Apr 25, 2026 - 13:50:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 92 : backward pass time 11.52 +[Apr 25, 2026 - 13:50:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 92 : optimizer step time 0.98 +[Apr 25, 2026 - 13:50:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 92-------------------- +[Apr 25, 2026 - 13:50:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0138 iter count: 1.0 +[Apr 25, 2026 - 13:50:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0138 iter count: 1.0 +[Apr 25, 2026 - 13:50:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:50:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:51:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810266 38.843746 38.780674 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.721191 67.869141 67.671875 +util 95.500000 99.000000 92.000000 +[Apr 25, 2026 - 13:51:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 93 : avg iteration time 49.87 seconds +[Apr 25, 2026 - 13:51:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 93 : data loading time 0.00 +[Apr 25, 2026 - 13:51:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 93 : avg forward pass time 12.92 +[Apr 25, 2026 - 13:51:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 93 : backward pass time 11.53 +[Apr 25, 2026 - 13:51:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 93 : optimizer step time 0.96 +[Apr 25, 2026 - 13:51:22 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 93-------------------- +[Apr 25, 2026 - 13:51:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0144 iter count: 1.0 +[Apr 25, 2026 - 13:51:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0144 iter count: 1.0 +[Apr 25, 2026 - 13:51:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:51:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:52:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810275 38.843746 38.780708 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.882324 67.906250 67.810547 +util 96.000000 100.000000 92.000000 +[Apr 25, 2026 - 13:52:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 94 : avg iteration time 46.91 seconds +[Apr 25, 2026 - 13:52:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 94 : data loading time 0.00 +[Apr 25, 2026 - 13:52:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 94 : avg forward pass time 11.43 +[Apr 25, 2026 - 13:52:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 94 : backward pass time 11.54 +[Apr 25, 2026 - 13:52:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 94 : optimizer step time 0.96 +[Apr 25, 2026 - 13:52:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 94-------------------- +[Apr 25, 2026 - 13:52:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0154 iter count: 1.0 +[Apr 25, 2026 - 13:52:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0154 iter count: 1.0 +[Apr 25, 2026 - 13:52:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:52:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810437 38.844456 38.780663 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.167480 133.191406 133.095703 +util 72.750000 78.000000 68.000000 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 95 : avg iteration time 169.04 seconds +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 95 : data loading time 0.00 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 95 : avg forward pass time 74.82 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 95 : backward pass time 9.13 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 95 : optimizer step time 0.32 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 95-------------------- +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 2320106.2008 iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.7058 iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 2320106.2008iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.6397iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.2933 iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 9.1182 iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 2607733.5000iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 13.2839iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 83166952.0000iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.7058iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.0885 iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 65.1732 iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.7655 iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 252.3957iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.9947 iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 2949.7659iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.5915 iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1316.9742iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0162 iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:54:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 13:55:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810444 38.844460 38.780685 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.945801 68.064453 67.906250 +util 97.000000 99.000000 91.000000 +[Apr 25, 2026 - 13:55:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 96 : avg iteration time 48.36 seconds +[Apr 25, 2026 - 13:55:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 96 : data loading time 0.00 +[Apr 25, 2026 - 13:55:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 96 : avg forward pass time 12.15 +[Apr 25, 2026 - 13:55:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 96 : backward pass time 11.53 +[Apr 25, 2026 - 13:55:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 96 : optimizer step time 0.99 +[Apr 25, 2026 - 13:55:47 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 96-------------------- +[Apr 25, 2026 - 13:55:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0159 iter count: 1.0 +[Apr 25, 2026 - 13:55:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0159 iter count: 1.0 +[Apr 25, 2026 - 13:55:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:55:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:56:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810438 38.844460 38.780685 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.955566 67.986328 67.945312 +util 95.250000 99.000000 91.000000 +[Apr 25, 2026 - 13:56:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 97 : avg iteration time 49.98 seconds +[Apr 25, 2026 - 13:56:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 97 : data loading time 0.00 +[Apr 25, 2026 - 13:56:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 97 : avg forward pass time 12.91 +[Apr 25, 2026 - 13:56:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 97 : backward pass time 11.61 +[Apr 25, 2026 - 13:56:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 97 : optimizer step time 1.00 +[Apr 25, 2026 - 13:56:37 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 97-------------------- +[Apr 25, 2026 - 13:56:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0175 iter count: 1.0 +[Apr 25, 2026 - 13:56:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0175 iter count: 1.0 +[Apr 25, 2026 - 13:56:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:56:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:57:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810346 38.844093 38.780663 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.970215 67.986328 67.964844 +util 98.750000 100.000000 98.000000 +[Apr 25, 2026 - 13:57:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 98 : avg iteration time 49.97 seconds +[Apr 25, 2026 - 13:57:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 98 : data loading time 0.00 +[Apr 25, 2026 - 13:57:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 98 : avg forward pass time 12.93 +[Apr 25, 2026 - 13:57:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 98 : backward pass time 11.58 +[Apr 25, 2026 - 13:57:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 98 : optimizer step time 0.98 +[Apr 25, 2026 - 13:57:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 98-------------------- +[Apr 25, 2026 - 13:57:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0179 iter count: 1.0 +[Apr 25, 2026 - 13:57:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0179 iter count: 1.0 +[Apr 25, 2026 - 13:57:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:57:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 13:58:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810352 38.844093 38.780708 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.862793 67.945312 67.615234 +util 97.000000 99.000000 95.000000 +[Apr 25, 2026 - 13:58:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 99 : avg iteration time 49.88 seconds +[Apr 25, 2026 - 13:58:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 99 : data loading time 0.00 +[Apr 25, 2026 - 13:58:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 99 : avg forward pass time 12.91 +[Apr 25, 2026 - 13:58:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 99 : backward pass time 11.54 +[Apr 25, 2026 - 13:58:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 99 : optimizer step time 0.97 +[Apr 25, 2026 - 13:58:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 99-------------------- +[Apr 25, 2026 - 13:58:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0179 iter count: 1.0 +[Apr 25, 2026 - 13:58:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0179 iter count: 1.0 +[Apr 25, 2026 - 13:58:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 13:58:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810359 38.844116 38.780689 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.134766 133.134766 133.134766 +util 68.500000 73.000000 64.000000 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 100 : avg iteration time 178.36 seconds +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 100 : data loading time 0.00 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 100 : avg forward pass time 79.91 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 100 : backward pass time 9.14 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 100 : optimizer step time 0.32 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 100-------------------- +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 24496.1554 iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6105 iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 24496.1554iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.0771iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.8115 iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.4515 iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 42915.6133iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 5.0684 iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 401531.7188iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6105iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.9293 iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 2588.1284iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.6460 iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 10850.2617iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.7298 iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 12829.8018iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.6147 iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 75316.6094iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0177 iter count: 1.0 +[Apr 25, 2026 - 14:01:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 25, 2026 - 14:01:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 18.81 seconds +[Apr 25, 2026 - 14:01:34 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 100 +[Apr 25, 2026 - 14:01:34 | INFO | fastgen.utils.checkpointer:save:301 ] Saving FSDP model to prefix /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100 +Traceback (most recent call last): + File "/home/work/.local/hyunbin/FastGen-redmd/train.py", line 46, in + main(config) + File "/home/work/.local/hyunbin/FastGen-redmd/train.py", line 37, in main + fastgen_trainer.run(model) + File "/home/work/.local/hyunbin/FastGen-redmd/fastgen/trainer.py", line 208, in run + latest_checkpoint_path = self.save_checkpoint(model, iter_cur) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/hyunbin/FastGen-redmd/fastgen/trainer.py", line 269, in save_checkpoint + path = self.checkpointer.save( + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/hyunbin/FastGen-redmd/fastgen/utils/checkpointer.py", line 314, in save + dcp.save(optim_state_dict, storage_writer=storage_writer) + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/logger.py", line 90, in wrapper + result = func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/utils.py", line 483, in inner_func + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/state_dict_saver.py", line 195, in save + return _save_state_dict( + ^^^^^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/state_dict_saver.py", line 464, in _save_state_dict + central_plan = distW.reduce_scatter("plan", local_step, global_step) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/utils.py", line 221, in reduce_scatter + raise result +torch.distributed.checkpoint.api.CheckpointException: CheckpointException ranks:dict_keys([0, 1, 2, 3]) +Traceback (most recent call last): (RANK 0) + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/utils.py", line 193, in reduce_scatter + local_data = map_fun() + ^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/logger.py", line 90, in wrapper + result = func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/state_dict_saver.py", line 449, in local_step + local_plan = storage_writer.prepare_local_plan(local_plan) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/filesystem.py", line 657, in prepare_local_plan + self.fs.mkdir(self.path) + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/filesystem.py", line 547, in mkdir + path.mkdir(parents=True, exist_ok=True) + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/pathlib.py", line 1311, in mkdir + os.mkdir(self, mode) +OSError: [Errno 28] No space left on device: '/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.fake_score_optim' +Traceback (most recent call last): (RANK 1) + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/utils.py", line 193, in reduce_scatter + local_data = map_fun() + ^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/logger.py", line 90, in wrapper + result = func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/state_dict_saver.py", line 449, in local_step + local_plan = storage_writer.prepare_local_plan(local_plan) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/filesystem.py", line 657, in prepare_local_plan + self.fs.mkdir(self.path) + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/filesystem.py", line 547, in mkdir + path.mkdir(parents=True, exist_ok=True) + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/pathlib.py", line 1311, in mkdir + os.mkdir(self, mode) +OSError: [Errno 28] No space left on device: '/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.fake_score_optim' +Traceback (most recent call last): (RANK 2) + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/utils.py", line 193, in reduce_scatter + local_data = map_fun() + ^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/logger.py", line 90, in wrapper + result = func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/state_dict_saver.py", line 449, in local_step + local_plan = storage_writer.prepare_local_plan(local_plan) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/filesystem.py", line 657, in prepare_local_plan + self.fs.mkdir(self.path) + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/filesystem.py", line 547, in mkdir + path.mkdir(parents=True, exist_ok=True) + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/pathlib.py", line 1311, in mkdir + os.mkdir(self, mode) +OSError: [Errno 28] No space left on device: '/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.fake_score_optim' +Traceback (most recent call last): (RANK 3) + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/utils.py", line 193, in reduce_scatter + local_data = map_fun() + ^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/logger.py", line 90, in wrapper + result = func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/state_dict_saver.py", line 449, in local_step + local_plan = storage_writer.prepare_local_plan(local_plan) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/filesystem.py", line 657, in prepare_local_plan + self.fs.mkdir(self.path) + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/filesystem.py", line 547, in mkdir + path.mkdir(parents=True, exist_ok=True) + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/pathlib.py", line 1311, in mkdir + os.mkdir(self, mode) +OSError: [Errno 28] No space left on device: '/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.fake_score_optim' + +[rank0]: Traceback (most recent call last): +[rank0]: File "/home/work/.local/hyunbin/FastGen-redmd/train.py", line 46, in +[rank0]: main(config) +[rank0]: File "/home/work/.local/hyunbin/FastGen-redmd/train.py", line 37, in main +[rank0]: fastgen_trainer.run(model) +[rank0]: File "/home/work/.local/hyunbin/FastGen-redmd/fastgen/trainer.py", line 208, in run +[rank0]: latest_checkpoint_path = self.save_checkpoint(model, iter_cur) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/hyunbin/FastGen-redmd/fastgen/trainer.py", line 269, in save_checkpoint +[rank0]: path = self.checkpointer.save( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/hyunbin/FastGen-redmd/fastgen/utils/checkpointer.py", line 314, in save +[rank0]: dcp.save(optim_state_dict, storage_writer=storage_writer) +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/logger.py", line 90, in wrapper +[rank0]: result = func(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/utils.py", line 483, in inner_func +[rank0]: return func(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/state_dict_saver.py", line 195, in save +[rank0]: return _save_state_dict( +[rank0]: ^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/state_dict_saver.py", line 464, in _save_state_dict +[rank0]: central_plan = distW.reduce_scatter("plan", local_step, global_step) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/utils.py", line 221, in reduce_scatter +[rank0]: raise result +[rank0]: torch.distributed.checkpoint.api.CheckpointException: CheckpointException ranks:dict_keys([0, 1, 2, 3]) +[rank0]: Traceback (most recent call last): (RANK 0) +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/utils.py", line 193, in reduce_scatter +[rank0]: local_data = map_fun() +[rank0]: ^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/logger.py", line 90, in wrapper +[rank0]: result = func(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/state_dict_saver.py", line 449, in local_step +[rank0]: local_plan = storage_writer.prepare_local_plan(local_plan) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/filesystem.py", line 657, in prepare_local_plan +[rank0]: self.fs.mkdir(self.path) +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/filesystem.py", line 547, in mkdir +[rank0]: path.mkdir(parents=True, exist_ok=True) +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/pathlib.py", line 1311, in mkdir +[rank0]: os.mkdir(self, mode) +[rank0]: OSError: [Errno 28] No space left on device: '/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.fake_score_optim' +[rank0]: Traceback (most recent call last): (RANK 1) +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/utils.py", line 193, in reduce_scatter +[rank0]: local_data = map_fun() +[rank0]: ^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/logger.py", line 90, in wrapper +[rank0]: result = func(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/state_dict_saver.py", line 449, in local_step +[rank0]: local_plan = storage_writer.prepare_local_plan(local_plan) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/filesystem.py", line 657, in prepare_local_plan +[rank0]: self.fs.mkdir(self.path) +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/filesystem.py", line 547, in mkdir +[rank0]: path.mkdir(parents=True, exist_ok=True) +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/pathlib.py", line 1311, in mkdir +[rank0]: os.mkdir(self, mode) +[rank0]: OSError: [Errno 28] No space left on device: '/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.fake_score_optim' +[rank0]: Traceback (most recent call last): (RANK 2) +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/utils.py", line 193, in reduce_scatter +[rank0]: local_data = map_fun() +[rank0]: ^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/logger.py", line 90, in wrapper +[rank0]: result = func(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/state_dict_saver.py", line 449, in local_step +[rank0]: local_plan = storage_writer.prepare_local_plan(local_plan) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/filesystem.py", line 657, in prepare_local_plan +[rank0]: self.fs.mkdir(self.path) +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/filesystem.py", line 547, in mkdir +[rank0]: path.mkdir(parents=True, exist_ok=True) +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/pathlib.py", line 1311, in mkdir +[rank0]: os.mkdir(self, mode) +[rank0]: OSError: [Errno 28] No space left on device: '/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.fake_score_optim' +[rank0]: Traceback (most recent call last): (RANK 3) +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/utils.py", line 193, in reduce_scatter +[rank0]: local_data = map_fun() +[rank0]: ^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/logger.py", line 90, in wrapper +[rank0]: result = func(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/state_dict_saver.py", line 449, in local_step +[rank0]: local_plan = storage_writer.prepare_local_plan(local_plan) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/filesystem.py", line 657, in prepare_local_plan +[rank0]: self.fs.mkdir(self.path) +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/distributed/checkpoint/filesystem.py", line 547, in mkdir +[rank0]: path.mkdir(parents=True, exist_ok=True) +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/pathlib.py", line 1311, in mkdir +[rank0]: os.mkdir(self, mode) +[rank0]: OSError: [Errno 28] No space left on device: '/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100.fake_score_optim' diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/requirements.txt b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/wandb-metadata.json b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..68d18b87479defa408b1eecc15c021c6559892d0 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/wandb-metadata.json @@ -0,0 +1,73 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-04-25T02:54:05.241185Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_sink1_window7_redmd_beta2_taew.py", + "-", + "trainer.resume=False", + "model.fake_score_optimizer.lr=3e-6", + "log_config.group=omniavatar_sf_audiofix", + "log_config.name=sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho", + "model.net.sink_size=3", + "model.net.local_attn_size=9" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "31752c6328ad798bdbb8f20a8372fca5fb35b990" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "1275297685504" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "bfyevs0fdc4ubo27edzw6b12a0hspzmh" +} \ No newline at end of file diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/wandb-summary.json b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..43baeeb8dff55dafd40ef76c5a94d552b981eb5e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/vsd_loss_weighted":24496.155395507812,"train/reward_weight_r2":12829.8017578125,"train/gan_loss_gen":0,"optimizer/iteration":100,"train/reward_weight_r0":2588.12841796875,"optimizer/lr_net":2e-06,"fake_score/local_total_params":[399300904,399021352,399021352,399021352],"train/vsd_loss":0.6104736328125,"train/reward_sync_c_r3":5.614727973937988,"model/total_params":17325462112,"profiler/optimizer_step_time":0.32211960293352604,"_runtime":7665.655896502,"_timestamp":1.7770932938514938e+09,"train/reward_sync_c_r0":3.92934513092041,"train/reward_sync_c_mean":4.077106475830078,"model/local_trainable_params":[420420640,420144160,420144160,420144160],"net/total_params":1421383840,"train/reward_weight_r1":10850.26171875,"optimizer/non_finite_grads_count (model_key net)":0,"train/total_loss":24496.155395507812,"profiler/backward_pass_time":9.139293359592557,"train/reward_sync_c_max":6.451520919799805,"fake_score/total_params":1596364960,"net/trainable_params":1421383840,"_wandb":{"runtime":7665},"train/reward_weight_max":401531.71875,"train/fake_score_loss":0.0176849365234375,"optimizer/lr_fake_score":3e-06,"train/reward_sync_c_r1":4.64597225189209,"model/trainable_params":1596364960,"profiler/data_loading_time":0.0008111707866191864,"net/local_total_params":[376675360,376398880,376398880,376398880],"train_media/data/real":{"path":"media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4","_type":"video-file","sha256":"47403c800591fce4a99b38622b7607baeec11f4e025df3962d292acf880fba6c","size":192248},"net/local_trainable_params":[376675360,376398880,376398880,376398880],"train/reward_sync_c_min":0.8115119934082031,"profiler/avg_forward_pass_time":79.91080007515848,"train/reward_weight_r3":75316.609375,"model/local_total_params":[4364057584,4362262512,4362262512,4362262512],"train/reward_sync_c_r2":4.729763031005859,"profiler/avg_iteration_time":178.3639998435974,"_step":100,"train/gan_loss_disc":0,"train/reward_weight_mean":42915.61328125,"optimizer/grad_norm (model_key net)":236946.359375,"fake_score/trainable_params":174981120,"fake_score/local_trainable_params":[43745280,43745280,43745280,43745280],"train_media/student/generation":{"size":257128,"path":"media/videos/train_media/student/generation_100_04a9b61f163a19decce9.mp4","_type":"video-file","sha256":"04a9b61f163a19decce96a0fd80161cd6c80a689cf81620d9c2d832ab2556052"},"train/vsd_loss_unweighted":0.6104736328125,"train/reward_weight_min":5.068394184112549} \ No newline at end of file diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/logs/debug-core.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..8d01c7192ca54ed76739bf3a1bcd58bdfd75347b --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2026-04-25T11:54:05.299037405+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp4w4sr92z/port-804038.txt","pid":804038,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-04-25T11:54:05.299535386+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":804038} +{"time":"2026-04-25T11:54:05.299524087+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-804038-811491-2506134228/socket","Net":"unix"}} +{"time":"2026-04-25T11:54:05.485783744+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-04-25T11:54:05.500512719+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"lxlsugeu","id":"1(@)"} +{"time":"2026-04-25T11:54:05.920154713+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"lxlsugeu","id":"1(@)"} +{"time":"2026-04-25T11:54:11.826226506+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"xy607skwizrv"} +{"time":"2026-04-25T14:01:52.228218917+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-04-25T14:01:52.228268685+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-04-25T14:01:52.228263131+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-04-25T14:01:52.228324461+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-04-25T14:01:52.228333322+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-804038-811491-2506134228/socket","Net":"unix"}} +{"time":"2026-04-25T14:01:53.757974182+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-04-25T14:01:53.757998511+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-04-25T14:01:53.758008998+09:00","level":"INFO","msg":"server is closed"} diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/logs/debug-internal.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..65cdd310e9498240d20c56c03a8a6e189a7cb379 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-04-25T11:54:05.500601086+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-04-25T11:54:05.920029696+09:00","level":"INFO","msg":"stream: created new stream","id":"lxlsugeu"} +{"time":"2026-04-25T11:54:05.920077898+09:00","level":"INFO","msg":"handler: started","stream_id":"lxlsugeu"} +{"time":"2026-04-25T11:54:05.920148525+09:00","level":"INFO","msg":"stream: started","id":"lxlsugeu"} +{"time":"2026-04-25T11:54:05.920172962+09:00","level":"INFO","msg":"writer: started","stream_id":"lxlsugeu"} +{"time":"2026-04-25T11:54:05.92017244+09:00","level":"INFO","msg":"sender: started","stream_id":"lxlsugeu"} +{"time":"2026-04-25T14:01:52.228267889+09:00","level":"INFO","msg":"stream: closing","id":"lxlsugeu"} +{"time":"2026-04-25T14:01:53.357668726+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-04-25T14:01:53.757464572+09:00","level":"INFO","msg":"handler: closed","stream_id":"lxlsugeu"} +{"time":"2026-04-25T14:01:53.757585931+09:00","level":"INFO","msg":"sender: closed","stream_id":"lxlsugeu"} +{"time":"2026-04-25T14:01:53.75759594+09:00","level":"INFO","msg":"stream: closed","id":"lxlsugeu"} diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/logs/debug.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3a67c835745921637578a223a732058a71ba8c45 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/logs/debug.log @@ -0,0 +1,21 @@ +2026-04-25 11:54:05,242 INFO MainThread:804038 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-04-25 11:54:05,242 INFO MainThread:804038 [wandb_setup.py:_flush():81] Configure stats pid to 804038 +2026-04-25 11:54:05,242 INFO MainThread:804038 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-25 11:54:05,242 INFO MainThread:804038 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/logs/debug.log +2026-04-25 11:54:05,242 INFO MainThread:804038 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/logs/debug-internal.log +2026-04-25 11:54:05,242 INFO MainThread:804038 [wandb_init.py:init():844] calling init triggers +2026-04-25 11:54:05,242 INFO MainThread:804038 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_audiofix', 'name': 'sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_100iter/checkpoints/0000100.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '5000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': None, 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': None, 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '9', 'sink_size': '3', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'False', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.833', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '3e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'True', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '2', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-04-25 11:54:05,242 INFO MainThread:804038 [wandb_init.py:init():892] starting backend +2026-04-25 11:54:05,486 INFO MainThread:804038 [wandb_init.py:init():895] sending inform_init request +2026-04-25 11:54:05,498 INFO MainThread:804038 [wandb_init.py:init():903] backend started and connected +2026-04-25 11:54:05,502 INFO MainThread:804038 [wandb_init.py:init():973] updated telemetry +2026-04-25 11:54:05,519 INFO MainThread:804038 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-25 11:54:06,570 INFO MainThread:804038 [wandb_init.py:init():1042] starting run threads in backend +2026-04-25 11:54:06,820 INFO MainThread:804038 [wandb_run.py:_console_start():2524] atexit reg +2026-04-25 11:54:06,820 INFO MainThread:804038 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-04-25 11:54:06,820 INFO MainThread:804038 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-04-25 11:54:06,820 INFO MainThread:804038 [wandb_run.py:_redirect():2465] Redirects installed. +2026-04-25 11:54:06,825 INFO MainThread:804038 [wandb_init.py:init():1082] run started, returning control to user process +2026-04-25 14:01:52,228 INFO wandb-AsyncioManager-main:804038 [service_client.py:_forward_responses():134] Reached EOF. +2026-04-25 14:01:52,228 INFO wandb-AsyncioManager-main:804038 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/run-lxlsugeu.wandb b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/run-lxlsugeu.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4cf441033a9ab1140b2aefebff09a1380605d446 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_115405-lxlsugeu/run-lxlsugeu.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2df0ff3d50b01f99fdeb8fd40eac258487346806980c524bb15634407b519013 +size 1728388 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/config.yaml b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86ba26750c7d53d0d527de993694c14b46acaf7d --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/config.yaml @@ -0,0 +1,389 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + d8umwtq7xb7xzsinoavf8csm2fyls44w: + args: + - --config=fastgen/configs/experiments/OmniAvatar/config_sf_sink1_window7_redmd_beta2_taew.py + - '-' + - trainer.resume=False + - model.fake_score_optimizer.lr=3e-6 + - log_config.group=omniavatar_sf_audiofix + - log_config.name=sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100 + - log_config.project=OmniAvatar-FastGen + - log_config.wandb_entity=paulhcho + - model.net.sink_size=3 + - model.net.local_attn_size=9 + codePath: train.py + codePathLocal: train.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.9" + disk: + /: + total: "1356758433792" + used: "1097760292864" + email: paul.hyunbin@gmail.com + executable: /home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12 + git: + commit: 31752c6328ad798bdbb8f20a8372fca5fb35b990 + remote: https://paulcho98:@github.com/paulcho98/FastGen.git + gpu: NVIDIA H200 + gpu_count: 4 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0 + host: main1 + memory: + total: "2163961778176" + os: Linux-5.15.0-151-generic-x86_64-with-glibc2.39 + program: /home/work/.local/hyunbin/FastGen-redmd/train.py + python: CPython 3.12.12 + root: /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100 + startedAt: "2026-04-25T05:26:02.550147Z" + writerId: d8umwtq7xb7xzsinoavf8csm2fyls44w + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "3": + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.12.12 + "5": 0.25.0 + "6": 5.3.0 + "12": 0.25.0 + "13": linux-x86_64 +dataloader_train: + value: + _target_: + batch_size: "8" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: "True" + neg_text_emb_path: null + num_workers: "2" + raw_audio_fps: "25.0" + raw_audio_num_frames: "81" + raw_audio_sample_rate: "16000" + use_ref_sequence: "True" +dataloader_val: + value: + _target_: + batch_size: "1" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: "False" + neg_text_emb_path: null + num_workers: "2" + use_ref_sequence: "True" +eval: + value: + max_ckpt: "100000000" + min_ckpt: "0" + num_samples: "50000" + samples_dir: samples + save_images: "False" +log_config: + value: + group: omniavatar_sf_audiofix + name: sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100 + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + value: + add_teacher_to_fsdp_dict: "True" + center_reward: "False" + clamp_reward: null + context_noise: "0.0" + ddp_find_unused_parameters: "True" + device: cuda + discriminator: + _target_: + all_res: + - "32" + - "16" + - "8" + feature_indices: '{0, 1, 2}' + in_channels: "256" + discriminator_optimizer: + _target_: + betas: + - "0.9" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "0.0001" + model: null + optim_type: adamw + weight_decay: "0.01" + discriminator_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + enable_gradient_in_rollout: "True" + enable_preprocessors: "True" + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: "65" + merge_lora: "False" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: "True" + fake_score_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "3e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + fsdp_meta_init: "False" + gan_loss_weight_gen: "0" + gan_r1_reg_alpha: "0.1" + gan_r1_reg_weight: "0.0" + gan_use_same_t_noise: "False" + grad_scaler_enabled: "False" + grad_scaler_growth_interval: "2000" + grad_scaler_init_scale: "65536.0" + guidance_scale: "4.5" + input_shape: + - "16" + - "21" + - "64" + - "64" + last_step_only: "False" + load_student_weights: "False" + net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: "3" + in_dim: "65" + local_attn_size: "9" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: "3" + total_num_frames: "21" + use_audio: "True" + use_dynamic_rope: "True" + net_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + net_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: "" + pretrained_student_net_path: "" + reward: + audio_sample_rate: "16000" + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: "True" + input_fps: "25.0" + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: "15" + reward_beta: "2" + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: "True" + sample_t_cfg: + log_t_df: "0.01" + max_t: "0.999" + min_t: "0.001" + shift: "5.0" + t_list: + - "0.999" + - "0.833" + - "0.0" + time_dist_type: shifted + train_p_mean: "-1.1" + train_p_std: "2.0" + save_reward_debug_video: "False" + skip_layers: null + start_gradient_frame: "0" + student_sample_steps: "2" + student_sample_type: sde + student_update_freq: "5" + teacher: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: "True" + timestep_cfg: + enabled: "True" + t_hi: "0.882" + t_lo: "0.556" + use_ema: "False" +model_class: + value: + _target_: + config: null +trainer: + value: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: "0.9999" + ema_halflife_kimg: "500" + ema_rampup_ratio: "0.05" + gamma: "16.97" + start_iter: "0" + type: constant + gpu_stats: + _target_: + every_n: "100" + grad_clip: + _target_: + grad_norm: "10.0" + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: "100" + wandb: + _target_: + fps: "25" + sample_logging_iter: "100" + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_100iter/checkpoints/0000100.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints + use_s3: "False" + cudnn: + benchmark: "True" + deterministic: "False" + ddp: "False" + fsdp: "True" + fsdp_cpu_offload: "False" + fsdp_min_num_params: "10000000" + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: "2" + logging_iter: "1" + max_iter: "5000" + offload_module_in_decoding: "False" + resume: "False" + save_ckpt_iter: "100" + seed: "0" + skip_initial_validation: "True" + tf32_enabled: "True" + val_seed: null + validation_iter: "100" + visualize_teacher: "False" diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23f2f377ccf704bcaa09a4c9b1a9b5151ab73aac --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47403c800591fce4a99b38622b7607baeec11f4e025df3962d292acf880fba6c +size 192248 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c0fdfbad50d1423fb0ee98a75767d17d3c13b75b --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e05a85e239f00a00e386c8d8a9eaabeac26dd4943226f41d80d9f6b7c50bff6 +size 181830 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1fd7a958d162e551e0f6ce9b94e075ce8f98807e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97779d11e00d4fea84a00b1667ca4d081ba87b85cf65fac40d228e9774f762d2 +size 194172 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9684c7bde9ed18086d39e7a9880b642c505b3947 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:112c0905448a821974f0bf0a94d9107473bb02b60e0ebe1e75a57d309670facb +size 219311 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e38036c10e966c2c82fa717b29431a1ff88e1ae --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2594e32c120a620ede9d4613c87c8985d5ec03a9250077d94093cf8001922aae +size 190461 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_100_cefe94f8f6eedfc62861.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_100_cefe94f8f6eedfc62861.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a6ef8f49aca15a90b2cd6a5fdd3a5de4c29d89b1 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_100_cefe94f8f6eedfc62861.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cefe94f8f6eedfc62861dd30a16f3dedbc1bf34316143a06b2e4ef425dd975c0 +size 255633 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_200_522fcaa3ca88be5f5888.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_200_522fcaa3ca88be5f5888.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..80006a3bc232f1606db2ec64f39ec0ceac6b25bb --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_200_522fcaa3ca88be5f5888.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:522fcaa3ca88be5f588861e123524ff3e72ce0438e9b79fe3f30f9cae5eade34 +size 236893 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_300_808c915bfaced95fd333.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_300_808c915bfaced95fd333.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bc59c6a5d50e259a1e6fe007273251a2794a35cd --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_300_808c915bfaced95fd333.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:808c915bfaced95fd3332725dcfccbb568a363f64e94edc08996d285c5f6f3dc +size 293157 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_400_34bbda270541d22aac7e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_400_34bbda270541d22aac7e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e2da7a8fa1bea1ee6f62c7d2bc944cfa21b5e9e5 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_400_34bbda270541d22aac7e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34bbda270541d22aac7e323a47cccabaf61a98df139f5a431bf804e92abf97c5 +size 353615 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_500_67ac3650ef016bc0d866.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_500_67ac3650ef016bc0d866.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bbbf106cd3d54661adeb4df76cf0fca1064b1acf --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/train_media/student/generation_500_67ac3650ef016bc0d866.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67ac3650ef016bc0d866bc3059b250b55dd12e4f8f3a8467c7d2b0a02ca1a353 +size 346634 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_0d36a47ac23df8c49717.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_0d36a47ac23df8c49717.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3c791411b0e2ae013359aae6ce0a43767991b888 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_0d36a47ac23df8c49717.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d36a47ac23df8c4971706e7fbaeb5f895b5387b07f787b5d618fb0e1c75a1c5 +size 277034 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_0f8b9fa5e2c061f798da.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_0f8b9fa5e2c061f798da.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d7514155ed88bd94179ad3d272fe792c4725aec3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_0f8b9fa5e2c061f798da.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f8b9fa5e2c061f798da5729d694149debbc2746e8546e04a34863fb99ffd5b7 +size 205790 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_2cd6ea1c10d36df608f4.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_2cd6ea1c10d36df608f4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..15121993664a2494472c74c9ed632ec1337aba6f --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_2cd6ea1c10d36df608f4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cd6ea1c10d36df608f4ddc7a79b61a2abc9ce85c167b7c660755dd7d2539b04 +size 231171 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_33508d91bb16115c4d95.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_33508d91bb16115c4d95.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..42b667aa9f49cf3bf47d97a293fef769f118ad14 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_33508d91bb16115c4d95.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33508d91bb16115c4d95e6198f7cc47cebdf10e0976832ab6256360aad9cc374 +size 233006 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_764975d52a365b7007c1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_764975d52a365b7007c1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5e6520b84e8ad4d2e13a395ac744752f0382e829 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_764975d52a365b7007c1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:764975d52a365b7007c11aaa27d23d5ab24b349037fd63ba261daff709a4994b +size 347095 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_777d1b756b1466b170dd.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_777d1b756b1466b170dd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..73d851320e736c02fb37f2338db0a6f89bdd4508 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_777d1b756b1466b170dd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:777d1b756b1466b170ddfa24695a24994aaa50e250fd05b1ba18992b0369f6ab +size 261060 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_78a815adf034840ead32.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_78a815adf034840ead32.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8bcf6e635824223b9f7d2d946a9fed0d47beea7b --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_78a815adf034840ead32.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78a815adf034840ead32ff2f1bfc8cf0812b3f1cf8759f4d9dc40b5afdeeb3a8 +size 275965 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_abc3526eae4ce7b95074.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_abc3526eae4ce7b95074.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..63a6a39b468cfee87035badf0a82ac610fe58387 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_abc3526eae4ce7b95074.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abc3526eae4ce7b950748fd01f274205ae4e57f9c2c1a81fd0302c54b09961a8 +size 307620 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_d1eda7c8bdee077184bd.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_d1eda7c8bdee077184bd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9352223907623d6ce239052ade3e23d6f683f052 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_d1eda7c8bdee077184bd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1eda7c8bdee077184bde159686e612d0462746af6e14ec4fe1a194a2f733ad6 +size 262289 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_ecf3b3bd0d2616edcd0d.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_ecf3b3bd0d2616edcd0d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e9541968248b421bf95ca2f0809fe512f2bf40b5 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_100_ecf3b3bd0d2616edcd0d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecf3b3bd0d2616edcd0d4134ac02e3c329a0d9bb19e02f2d7c7248bf84263a25 +size 346742 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_3ddf326081c0796f3a13.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_3ddf326081c0796f3a13.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0d234c65f847aa41b4c35b4a237b432ed90e8046 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_3ddf326081c0796f3a13.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ddf326081c0796f3a1379a4a390a475171c1e3d16b0faddbd4b4c337be61d92 +size 269350 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_434f141d2e23c05bd74d.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_434f141d2e23c05bd74d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1e686677effeabea575b0274777af24a0afd8f55 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_434f141d2e23c05bd74d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:434f141d2e23c05bd74df7d689764b2c850863193e3f588075801f1adf4cf967 +size 344105 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_4a28a1edd2355b9b361f.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_4a28a1edd2355b9b361f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3e67840e9ca5dcc7ee4c39b38bbe0e42f975110e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_4a28a1edd2355b9b361f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a28a1edd2355b9b361fc5c72f28233f30338824e3759b0dfaffc7ad917b092d +size 227478 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_4c53a95f289f22e345ce.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_4c53a95f289f22e345ce.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8764824a91f99932bbdf87c0d0785f11dbefad20 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_4c53a95f289f22e345ce.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c53a95f289f22e345ce3febff3154ddb6745a21a7414e919f65254e498cfab2 +size 272148 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_54b5ffc7e997ed3e03fe.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_54b5ffc7e997ed3e03fe.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7b49c64afb8eaeb95f98d9d753ebeca3744e970f --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_54b5ffc7e997ed3e03fe.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54b5ffc7e997ed3e03fe90c9a2c7ac89f806cc2a59701913d227f22e545bd48d +size 207825 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_683f35385b67a977e4ce.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_683f35385b67a977e4ce.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6407b7c39e0c09ec6da64777252c9f193a839488 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_683f35385b67a977e4ce.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:683f35385b67a977e4cef6f455c5d716d2371dc40860a09357157e6e556994aa +size 260726 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_7aef4ffdb8a10e17e5e9.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_7aef4ffdb8a10e17e5e9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4481c6b00f33578bcaaf16a81cd0e648cb66c21d --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_7aef4ffdb8a10e17e5e9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7aef4ffdb8a10e17e5e9386a346dcb108d82ac0c6969e3ee257bff15eec20731 +size 292098 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_857e12191ec0dfaedc56.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_857e12191ec0dfaedc56.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..69b1f43df22e05605f1ef6abf9845a3959f530de --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_857e12191ec0dfaedc56.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:857e12191ec0dfaedc56c104d3b799c206308f9eb6c0f2b36ac1c2afdb75d5b9 +size 254473 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_c97cf0d22a4f3855bf97.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_c97cf0d22a4f3855bf97.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..16923298039d02e2fd21af5b907f8bd2ba7520ad --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_c97cf0d22a4f3855bf97.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c97cf0d22a4f3855bf974724f24142c2a167cc0964fe669bf519115316edbd62 +size 331770 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_ed471b69e01df1aa7cd9.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_ed471b69e01df1aa7cd9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6ccbb5b30937bb106fa1234766a73f51c12f62a7 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_200_ed471b69e01df1aa7cd9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed471b69e01df1aa7cd90d72d10d73e985fd379703ef16f139f86afc4864af00 +size 234650 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_063074a6e3d9153b241b.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_063074a6e3d9153b241b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8700dea598ebd2cc27cfb7a86ed2f257e63163cc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_063074a6e3d9153b241b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:063074a6e3d9153b241b396173e4d87857a73a610fd39a66d847d30337d494b2 +size 232767 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_0cbdd8e709e49641cc46.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_0cbdd8e709e49641cc46.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..01a539b7687f17fc95a76a6ab3bd4d388eac0ed5 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_0cbdd8e709e49641cc46.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cbdd8e709e49641cc46eb02d2110a0962f3ec246e221f9a2db26632ce2a7ad3 +size 285970 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_1fb55a14d14f5a0848cd.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_1fb55a14d14f5a0848cd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..44f1493103bfcaf48e107903b724c6f17fefa459 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_1fb55a14d14f5a0848cd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fb55a14d14f5a0848cdfc9af9e1d941a9bfb049bc408b175dcf2523229e79fe +size 310090 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_7ed7e3c953e4f537d576.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_7ed7e3c953e4f537d576.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..53239b31f9a2d8ea1405b0dafba7816af4809d69 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_7ed7e3c953e4f537d576.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ed7e3c953e4f537d576f44d93de63e15982c201613d8d2f46c9b84ab64e5956 +size 381132 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_9b077cec2e4d4dc35c9a.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_9b077cec2e4d4dc35c9a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ecd9c2b05fedf6c1508252c59f3485e2a0c1afac --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_9b077cec2e4d4dc35c9a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b077cec2e4d4dc35c9a2798118840c60ad6e0a411e96f77941eed562eee3a5d +size 335818 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_ad6468cffc81cdef7dfc.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_ad6468cffc81cdef7dfc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..88515fd01b75d121d3b516d8f24f829e2c8b170e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_ad6468cffc81cdef7dfc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad6468cffc81cdef7dfca372b87c6888a5c3facee0640bda85271dc868d7d174 +size 303953 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_b0ea71e01862a49cf33f.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_b0ea71e01862a49cf33f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..62ec75c824e8a804eb84778310405d392e467081 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_b0ea71e01862a49cf33f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0ea71e01862a49cf33f630303e09395570a13e2cde06343a6cf091cf496346a +size 267195 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_bfeee8b5be654677567c.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_bfeee8b5be654677567c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8a85717d509dbf38446d25ca90dd19fa4f0eaf2d --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_bfeee8b5be654677567c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfeee8b5be654677567cbe71e950808108da82114d4fa6fa5406fa5f8bd91e07 +size 383613 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_d13343fbf8c221ef568f.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_d13343fbf8c221ef568f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..58f2094818d21ad2c8134fb0a3a8936e094ccbd9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_d13343fbf8c221ef568f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d13343fbf8c221ef568f085e234c4df292a83521291740a17ff389dae63d90dc +size 301901 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_fb20fb759d5b7e6408fd.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_fb20fb759d5b7e6408fd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f11d55ca390302363ad34244a78cb99c02ae5497 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_300_fb20fb759d5b7e6408fd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb20fb759d5b7e6408fd669e182e30f355411bdf1f55941a5bf92c6d7c50e4df +size 264081 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_27e288e1f9c7c94078bd.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_27e288e1f9c7c94078bd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9a7ad6b0ad70c29ff1909222c2d7242b3b1c82e0 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_27e288e1f9c7c94078bd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27e288e1f9c7c94078bd1cf170ac2049ade44c135c154ee294ad0662c59c1bdd +size 311136 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_2e45c60112dbc1dfddbb.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_2e45c60112dbc1dfddbb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4a1c0eb424707950fd2e2d00d306ceedeceb5690 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_2e45c60112dbc1dfddbb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e45c60112dbc1dfddbb21bd1c6adfd6614a8e7dc9c703c37b2dcec85ed3e9bc +size 295376 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_3cba6a3368b82ac19add.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_3cba6a3368b82ac19add.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ad80cd1e28268efd713af73664e6e530e0021cd2 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_3cba6a3368b82ac19add.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cba6a3368b82ac19add9f9145a975fe7f90ed166b67e7b61bfb508393e92576 +size 379587 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_5d2f0c3ef8f4f3d37d4a.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_5d2f0c3ef8f4f3d37d4a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..96201203255d236a5c76e5e75f62b589f572903d --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_5d2f0c3ef8f4f3d37d4a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d2f0c3ef8f4f3d37d4a0bf3ee03341075c08ac39322de9c50a5f1ba61faab04 +size 337451 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_91bc2d478d9b4731fdc5.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_91bc2d478d9b4731fdc5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..398cb2205937aed04f0b0f1ea1a093b799500df4 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_91bc2d478d9b4731fdc5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91bc2d478d9b4731fdc5ebaf2ebcbb3ea73699d780f70f980fda992eec30478b +size 352935 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_942baaa2ff6d38d3e1de.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_942baaa2ff6d38d3e1de.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d11c72a6515227132f6952b2b0c6206131fe96d8 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_942baaa2ff6d38d3e1de.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:942baaa2ff6d38d3e1de423798702eb1c3efac06a58c0dcc2ea1727f59c8c7f2 +size 441304 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_a7fea23d34df802b17c1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_a7fea23d34df802b17c1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ae3ba5a7aa89f39ed28d4dc835ad8398ca916b80 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_a7fea23d34df802b17c1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7fea23d34df802b17c186cb87f9fc32259b30600a67cc6b50dfc8ff616f668f +size 415477 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_ba74a72801db8ecf90b1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_ba74a72801db8ecf90b1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7ccba0e2e76e096415104852ca653858dd8b532e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_ba74a72801db8ecf90b1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba74a72801db8ecf90b12bc6729b3bf1341ca3122e49885a5ec6f519c5d3feeb +size 267374 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_eae35008e76a476df277.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_eae35008e76a476df277.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bc285570ee478c6e4a310d4571d8d420f79cc443 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_eae35008e76a476df277.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eae35008e76a476df277855aa5301a877e1f7da63e3f93e8ac3b9517ebfc2717 +size 334027 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_f37b8bd7ac458e989f64.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_f37b8bd7ac458e989f64.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4661b9618f32d8d3268254b99fb208c45d683447 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_400_f37b8bd7ac458e989f64.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f37b8bd7ac458e989f647deb178372ec8ccb4d843d48b1cd5c9954922dea4a1a +size 294812 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_08b64e302ec053e09f7a.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_08b64e302ec053e09f7a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cfa2f41cbd4d109eb4d3217d424a2ce0ef93d1b8 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_08b64e302ec053e09f7a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08b64e302ec053e09f7ad36f87a71ccc71a553a8f2445caf8724efe85e14ffbe +size 358062 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_364210d1d39487ed18dd.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_364210d1d39487ed18dd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1add2c4235fb70991a8a26344b9bdef77e2b91aa --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_364210d1d39487ed18dd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:364210d1d39487ed18dd123ed07f223616c35ace4fd3bdfab872a29b20c0aded +size 372058 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_382dcc9d28af35e517dd.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_382dcc9d28af35e517dd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fc842abff0c1b1d77d7ca63b43f8f70cf7c80db2 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_382dcc9d28af35e517dd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:382dcc9d28af35e517dd747d8e08ecf5123327ce5699fc118dbe1e53a4261191 +size 320950 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_415f82bb3b3e9cd393a0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_415f82bb3b3e9cd393a0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..af37630eacd0c3c688ee3509400bdf198f08ae92 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_415f82bb3b3e9cd393a0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:415f82bb3b3e9cd393a095c95503284e21ed8de81292a042499f9cc5a716a9be +size 405114 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_52ceb51262f49782f4b9.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_52ceb51262f49782f4b9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b4f1018a0cc81dc395170bd005486ce0f718bc74 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_52ceb51262f49782f4b9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52ceb51262f49782f4b91466896ee5be6c5c1c6e2e83f5f16137e78ff1278f9a +size 332242 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_90b0c03047027618dec6.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_90b0c03047027618dec6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..20fb6e1e2db42c176294c948959f5cfb6bee4c0f --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_90b0c03047027618dec6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90b0c03047027618dec650a0421971be47de5dbbfa44396f5e38eac4fdee9704 +size 477909 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_b22bbc588eb2412872cb.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_b22bbc588eb2412872cb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7ade40918f639ba8f63e2e37f5d71423f41fb8d3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_b22bbc588eb2412872cb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b22bbc588eb2412872cb360c8340ac354f0063cbffc58c5b8fc61407a74b6277 +size 442827 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_b6e3fb89f026804ddf55.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_b6e3fb89f026804ddf55.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..01233a7c829d1945d6ad7a1b4b12d15ddc49d5c5 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_b6e3fb89f026804ddf55.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6e3fb89f026804ddf554a8a2ca65808354af72d08ae6779be0ed727eaf08d00 +size 336122 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_e2e0efb6d3340c8a74b5.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_e2e0efb6d3340c8a74b5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e73411cade98e1d455b23322ddf253fb0e125b73 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_e2e0efb6d3340c8a74b5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2e0efb6d3340c8a74b58b9d1b68b7a050660b6658f2f789686d51bc520a4f9c +size 290175 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_f565d15af15a1ba1d0af.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_f565d15af15a1ba1d0af.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6dce44becdcee9fccd8e9d9fb6d04d6f08b12f9e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/generated_500_f565d15af15a1ba1d0af.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f565d15af15a1ba1d0af3054dfa7270010be5130908fc3e411426ef37525df7a +size 377778 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/output.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..995c8fd4e5eb792d544dcbeb7a24ebe7cbc44235 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/output.log @@ -0,0 +1,10758 @@ +[Apr 25, 2026 - 14:26:04 | INFO | fastgen.callbacks.wandb:init_wandb:259 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[Apr 25, 2026 - 14:26:04 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[Apr 25, 2026 - 14:26:04 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[Apr 25, 2026 - 14:26:04 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[Apr 25, 2026 - 14:26:04 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[Apr 25, 2026 - 14:26:04 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[Apr 25, 2026 - 14:26:04 | INFO | fastgen.trainer:run:77 ] Starting training +[Apr 25, 2026 - 14:26:04 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[Apr 25, 2026 - 14:26:04 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_100iter/checkpoints/0000100.pth +[Apr 25, 2026 - 14:26:08 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict... +[Apr 25, 2026 - 14:26:20 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: +[Apr 25, 2026 - 14:26:20 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_100iter/checkpoints/0000100.pth at iteration 100 +[Apr 25, 2026 - 14:26:20 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 100. +[Apr 25, 2026 - 14:26:20 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[Apr 25, 2026 - 14:27:26 | INFO | fastgen.methods.model:on_train_begin:296 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[Apr 25, 2026 - 14:27:26 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[Apr 25, 2026 - 14:27:26 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[Apr 25, 2026 - 14:27:26 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[Apr 25, 2026 - 14:27:26 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)... +[Apr 25, 2026 - 14:27:26 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2166 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[Apr 25, 2026 - 14:27:29 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[Apr 25, 2026 - 14:27:29 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 2.9s +[Apr 25, 2026 - 14:27:29 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.60B params)... +[Apr 25, 2026 - 14:27:29 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:774 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[Apr 25, 2026 - 14:27:32 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[Apr 25, 2026 - 14:27:32 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 2.8s +[Apr 25, 2026 - 14:27:32 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[Apr 25, 2026 - 14:27:32 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:774 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[Apr 25, 2026 - 14:27:45 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[Apr 25, 2026 - 14:27:45 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 13.1s +[Apr 25, 2026 - 14:27:45 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[Apr 25, 2026 - 14:27:45 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[Apr 25, 2026 - 14:27:45 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[Apr 25, 2026 - 14:27:45 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 0. +[Apr 25, 2026 - 14:27:45 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader... +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:362 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1 +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1 +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingReDMD) has 1596.36 M trainable and 17325.46 M total params (logical). +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.42 M trainable and 4364.06 M total params LOCAL on rank 0. +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.14 M trainable and 4362.26 M total params LOCAL on rank 1. +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.14 M trainable and 4362.26 M total params LOCAL on rank 2. +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.14 M trainable and 4362.26 M total params LOCAL on rank 3. +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.68 M trainable and 376.68 M total params LOCAL on rank 0. +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 1. +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 2. +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 3. +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 174.98 M trainable and 1596.36 M total params (logical). +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.30 M total params LOCAL on rank 0. +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 1. +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 2. +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 3. +[Apr 25, 2026 - 14:27:50 | INFO | fastgen.trainer:run:174 ] iter_start: 0 +[Apr 25, 2026 - 14:28:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.578676 38.620079 38.503593 +peak_gpu_mem_gb 60.567137 60.567137 60.567137 +peak_gpu_mem_reserved_gb 62.595703 62.595703 62.595703 +util 89.250000 94.000000 84.000000 +[Apr 25, 2026 - 14:28:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 1 : data loading time 0.97 +[Apr 25, 2026 - 14:28:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 1 : avg forward pass time 11.16 +[Apr 25, 2026 - 14:28:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 1 : backward pass time 11.52 +[Apr 25, 2026 - 14:28:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 1 : optimizer step time 1.07 +[Apr 25, 2026 - 14:28:39 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 1-------------------- +[Apr 25, 2026 - 14:28:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0171 iter count: 1.0 +[Apr 25, 2026 - 14:28:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0171 iter count: 1.0 +[Apr 25, 2026 - 14:28:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:28:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:29:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.754258 38.796349 38.678917 +peak_gpu_mem_gb 62.354346 62.354346 62.354346 +peak_gpu_mem_reserved_gb 64.042969 64.042969 64.042969 +util 97.500000 99.000000 96.000000 +[Apr 25, 2026 - 14:29:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 2 : avg iteration time 50.04 seconds +[Apr 25, 2026 - 14:29:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 2 : data loading time 0.00 +[Apr 25, 2026 - 14:29:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 2 : avg forward pass time 12.94 +[Apr 25, 2026 - 14:29:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 2 : backward pass time 11.57 +[Apr 25, 2026 - 14:29:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 2 : optimizer step time 0.99 +[Apr 25, 2026 - 14:29:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 2-------------------- +[Apr 25, 2026 - 14:29:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0139 iter count: 1.0 +[Apr 25, 2026 - 14:29:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[Apr 25, 2026 - 14:29:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:29:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:30:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.754032 38.794880 38.678993 +peak_gpu_mem_gb 62.354346 62.354346 62.354346 +peak_gpu_mem_reserved_gb 64.042969 64.042969 64.042969 +util 96.000000 99.000000 93.000000 +[Apr 25, 2026 - 14:30:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 3 : avg iteration time 52.96 seconds +[Apr 25, 2026 - 14:30:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 3 : data loading time 0.00 +[Apr 25, 2026 - 14:30:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 3 : avg forward pass time 14.42 +[Apr 25, 2026 - 14:30:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 3 : backward pass time 11.56 +[Apr 25, 2026 - 14:30:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 3 : optimizer step time 0.97 +[Apr 25, 2026 - 14:30:22 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 3-------------------- +[Apr 25, 2026 - 14:30:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0120 iter count: 1.0 +[Apr 25, 2026 - 14:30:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[Apr 25, 2026 - 14:30:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:30:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:31:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.754301 38.794910 38.679115 +peak_gpu_mem_gb 62.354346 62.354346 62.354346 +peak_gpu_mem_reserved_gb 64.077148 64.121094 64.062500 +util 95.500000 99.000000 92.000000 +[Apr 25, 2026 - 14:31:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 4 : avg iteration time 47.08 seconds +[Apr 25, 2026 - 14:31:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 4 : data loading time 0.00 +[Apr 25, 2026 - 14:31:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 4 : avg forward pass time 11.48 +[Apr 25, 2026 - 14:31:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 4 : backward pass time 11.54 +[Apr 25, 2026 - 14:31:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 4 : optimizer step time 0.99 +[Apr 25, 2026 - 14:31:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 4-------------------- +[Apr 25, 2026 - 14:31:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0177 iter count: 1.0 +[Apr 25, 2026 - 14:31:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0177 iter count: 1.0 +[Apr 25, 2026 - 14:31:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:31:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.817023 38.855206 38.742222 +peak_gpu_mem_gb 126.559253 126.559253 126.559253 +peak_gpu_mem_reserved_gb 138.151367 138.166016 138.107422 +util 62.250000 67.000000 58.000000 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 5 : avg iteration time 179.87 seconds +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 5 : data loading time 0.00 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 5 : avg forward pass time 79.90 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 5 : backward pass time 10.46 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 5 : optimizer step time 0.44 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 5-------------------- +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 19017.4134 iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6169 iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 19017.4134iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.2858iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5295 iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.8956 iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 34925.5977iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.8836 iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 975888.1875iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6169iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.7892 iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1955.4000iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.9428 iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2658.8088iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.4433 iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 978.9815iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 1.9511 iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 49.5076 iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0165 iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:34:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 14:35:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.817081 38.855381 38.742233 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.892090 67.966797 67.867188 +util 97.750000 99.000000 96.000000 +[Apr 25, 2026 - 14:35:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 6 : avg iteration time 53.59 seconds +[Apr 25, 2026 - 14:35:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 6 : data loading time 0.00 +[Apr 25, 2026 - 14:35:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 6 : avg forward pass time 14.76 +[Apr 25, 2026 - 14:35:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 6 : backward pass time 11.54 +[Apr 25, 2026 - 14:35:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 6 : optimizer step time 0.98 +[Apr 25, 2026 - 14:35:03 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 6-------------------- +[Apr 25, 2026 - 14:35:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 14:35:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 14:35:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:35:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:35:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.817715 38.857018 38.743118 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.921387 68.083984 67.867188 +util 97.500000 100.000000 95.000000 +[Apr 25, 2026 - 14:35:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 7 : avg iteration time 52.88 seconds +[Apr 25, 2026 - 14:35:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 7 : data loading time 0.00 +[Apr 25, 2026 - 14:35:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 7 : avg forward pass time 14.43 +[Apr 25, 2026 - 14:35:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 7 : backward pass time 11.53 +[Apr 25, 2026 - 14:35:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 7 : optimizer step time 0.97 +[Apr 25, 2026 - 14:35:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 7-------------------- +[Apr 25, 2026 - 14:35:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0125 iter count: 1.0 +[Apr 25, 2026 - 14:35:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[Apr 25, 2026 - 14:35:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:35:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:36:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.817722 38.857025 38.743141 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.877441 68.083984 67.808594 +util 97.250000 100.000000 95.000000 +[Apr 25, 2026 - 14:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 8 : avg iteration time 52.90 seconds +[Apr 25, 2026 - 14:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 8 : data loading time 0.00 +[Apr 25, 2026 - 14:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 8 : avg forward pass time 14.43 +[Apr 25, 2026 - 14:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 8 : backward pass time 11.53 +[Apr 25, 2026 - 14:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 8 : optimizer step time 0.97 +[Apr 25, 2026 - 14:36:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 8-------------------- +[Apr 25, 2026 - 14:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0116 iter count: 1.0 +[Apr 25, 2026 - 14:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[Apr 25, 2026 - 14:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:36:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:37:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.817783 38.857056 38.743286 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.892090 68.083984 67.828125 +util 96.750000 100.000000 94.000000 +[Apr 25, 2026 - 14:37:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 9 : avg iteration time 46.99 seconds +[Apr 25, 2026 - 14:37:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 9 : data loading time 0.00 +[Apr 25, 2026 - 14:37:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 9 : avg forward pass time 11.46 +[Apr 25, 2026 - 14:37:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 9 : backward pass time 11.54 +[Apr 25, 2026 - 14:37:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 9 : optimizer step time 0.98 +[Apr 25, 2026 - 14:37:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 9-------------------- +[Apr 25, 2026 - 14:37:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0153 iter count: 1.0 +[Apr 25, 2026 - 14:37:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0153 iter count: 1.0 +[Apr 25, 2026 - 14:37:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:37:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.818304 38.857189 38.743893 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.099121 133.115234 133.093750 +util 69.500000 78.000000 61.000000 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 10 : avg iteration time 175.97 seconds +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 10 : data loading time 0.00 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 10 : avg forward pass time 78.72 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 10 : backward pass time 9.14 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 10 : optimizer step time 0.34 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 10-------------------- +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 15470.9534 iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6161 iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 15470.9534iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.0560iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6737 iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.1242 iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 25172.5430iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.8476 iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 208664.0469iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6161iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.6102 iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 74633.6328iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.2802 iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 706.5853iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.1446 iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 3980.3137iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.1242 iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 208664.0469iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0156 iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:40:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 14:41:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.818314 38.857189 38.743900 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.955566 68.044922 67.925781 +util 96.250000 99.000000 93.000000 +[Apr 25, 2026 - 14:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 11 : avg iteration time 51.53 seconds +[Apr 25, 2026 - 14:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 11 : data loading time 0.00 +[Apr 25, 2026 - 14:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 11 : avg forward pass time 13.74 +[Apr 25, 2026 - 14:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 11 : backward pass time 11.53 +[Apr 25, 2026 - 14:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 11 : optimizer step time 0.98 +[Apr 25, 2026 - 14:41:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 11-------------------- +[Apr 25, 2026 - 14:41:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0137 iter count: 1.0 +[Apr 25, 2026 - 14:41:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0137 iter count: 1.0 +[Apr 25, 2026 - 14:41:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:41:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:42:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.818597 38.858284 38.743904 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.853027 67.886719 67.751953 +util 96.000000 99.000000 93.000000 +[Apr 25, 2026 - 14:42:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 12 : avg iteration time 52.80 seconds +[Apr 25, 2026 - 14:42:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 12 : data loading time 0.00 +[Apr 25, 2026 - 14:42:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 12 : avg forward pass time 14.36 +[Apr 25, 2026 - 14:42:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 12 : backward pass time 11.54 +[Apr 25, 2026 - 14:42:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 12 : optimizer step time 0.99 +[Apr 25, 2026 - 14:42:16 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 12-------------------- +[Apr 25, 2026 - 14:42:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0113 iter count: 1.0 +[Apr 25, 2026 - 14:42:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[Apr 25, 2026 - 14:42:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:42:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:43:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.818600 38.858284 38.743908 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.853027 67.906250 67.693359 +util 94.750000 99.000000 90.000000 +[Apr 25, 2026 - 14:43:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 13 : avg iteration time 52.78 seconds +[Apr 25, 2026 - 14:43:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 13 : data loading time 0.00 +[Apr 25, 2026 - 14:43:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 13 : avg forward pass time 14.38 +[Apr 25, 2026 - 14:43:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 13 : backward pass time 11.53 +[Apr 25, 2026 - 14:43:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 13 : optimizer step time 0.98 +[Apr 25, 2026 - 14:43:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 13-------------------- +[Apr 25, 2026 - 14:43:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0101 iter count: 1.0 +[Apr 25, 2026 - 14:43:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[Apr 25, 2026 - 14:43:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:43:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:43:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.818585 38.858047 38.743935 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.867676 67.925781 67.693359 +util 97.000000 100.000000 94.000000 +[Apr 25, 2026 - 14:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 14 : avg iteration time 49.94 seconds +[Apr 25, 2026 - 14:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 14 : data loading time 0.00 +[Apr 25, 2026 - 14:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 14 : avg forward pass time 12.94 +[Apr 25, 2026 - 14:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 14 : backward pass time 11.52 +[Apr 25, 2026 - 14:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 14 : optimizer step time 0.99 +[Apr 25, 2026 - 14:43:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 14-------------------- +[Apr 25, 2026 - 14:43:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0145 iter count: 1.0 +[Apr 25, 2026 - 14:43:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0145 iter count: 1.0 +[Apr 25, 2026 - 14:43:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:43:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.818670 38.858311 38.743942 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.125000 133.134766 133.095703 +util 67.500000 73.000000 62.000000 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 15 : avg iteration time 170.09 seconds +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 15 : data loading time 0.00 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 15 : avg forward pass time 75.78 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 15 : backward pass time 9.15 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 15 : optimizer step time 0.34 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 15-------------------- +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 8977.8019 iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5953 iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 8977.8019iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.4742iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6601 iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 5.9596 iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 13616.7627iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.7441 iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 150116.1562iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5953iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.1827 iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 4296.1670iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.3244 iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 5703.6025iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.1381 iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 71.9641 iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.2939 iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 5365.6582iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:46:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 14:47:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.818686 38.858368 38.743969 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.911621 67.964844 67.751953 +util 95.750000 100.000000 92.000000 +[Apr 25, 2026 - 14:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 16 : avg iteration time 48.49 seconds +[Apr 25, 2026 - 14:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 16 : data loading time 0.00 +[Apr 25, 2026 - 14:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 16 : avg forward pass time 12.23 +[Apr 25, 2026 - 14:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 16 : backward pass time 11.52 +[Apr 25, 2026 - 14:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 16 : optimizer step time 0.98 +[Apr 25, 2026 - 14:47:37 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 16-------------------- +[Apr 25, 2026 - 14:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0136 iter count: 1.0 +[Apr 25, 2026 - 14:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[Apr 25, 2026 - 14:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:47:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:48:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.818686 38.858341 38.743958 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.755371 68.064453 67.652344 +util 95.500000 100.000000 91.000000 +[Apr 25, 2026 - 14:48:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 17 : avg iteration time 50.00 seconds +[Apr 25, 2026 - 14:48:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 17 : data loading time 0.00 +[Apr 25, 2026 - 14:48:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 17 : avg forward pass time 12.96 +[Apr 25, 2026 - 14:48:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 17 : backward pass time 11.54 +[Apr 25, 2026 - 14:48:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 17 : optimizer step time 0.99 +[Apr 25, 2026 - 14:48:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 17-------------------- +[Apr 25, 2026 - 14:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0109 iter count: 1.0 +[Apr 25, 2026 - 14:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[Apr 25, 2026 - 14:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:48:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:49:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.818892 38.858341 38.743938 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.960449 68.064453 67.925781 +util 97.250000 100.000000 94.000000 +[Apr 25, 2026 - 14:49:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 18 : avg iteration time 49.94 seconds +[Apr 25, 2026 - 14:49:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 18 : data loading time 0.00 +[Apr 25, 2026 - 14:49:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 18 : avg forward pass time 12.93 +[Apr 25, 2026 - 14:49:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 18 : backward pass time 11.55 +[Apr 25, 2026 - 14:49:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 18 : optimizer step time 0.98 +[Apr 25, 2026 - 14:49:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 18-------------------- +[Apr 25, 2026 - 14:49:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 14:49:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 14:49:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:49:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:50:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.819196 38.858620 38.743961 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.770020 68.064453 67.671875 +util 96.750000 99.000000 94.000000 +[Apr 25, 2026 - 14:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 19 : avg iteration time 50.00 seconds +[Apr 25, 2026 - 14:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 19 : data loading time 0.00 +[Apr 25, 2026 - 14:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 19 : avg forward pass time 12.98 +[Apr 25, 2026 - 14:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 19 : backward pass time 11.52 +[Apr 25, 2026 - 14:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 19 : optimizer step time 1.00 +[Apr 25, 2026 - 14:50:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 19-------------------- +[Apr 25, 2026 - 14:50:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 14:50:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 14:50:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:50:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.819626 38.859016 38.744915 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.105957 133.115234 133.078125 +util 69.500000 78.000000 60.000000 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 20 : avg iteration time 175.16 seconds +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 20 : data loading time 0.00 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 20 : avg forward pass time 77.89 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 20 : backward pass time 9.13 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 20 : optimizer step time 0.34 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 20-------------------- +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 24613.9915 iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6835 iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 24613.9915iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.7692iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.6940 iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.7089 iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 48255.6836iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 29.6081iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 671875.0000iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6835iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.0161 iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 22745.9551iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.3277 iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 776.9726iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.1475 iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 29586.8125iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.6208 iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 563312.8750iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:53:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 14:53:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.819974 38.860382 38.744942 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.843262 67.906250 67.654297 +util 96.750000 100.000000 94.000000 +[Apr 25, 2026 - 14:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 21 : avg iteration time 48.53 seconds +[Apr 25, 2026 - 14:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 21 : data loading time 0.00 +[Apr 25, 2026 - 14:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 21 : avg forward pass time 12.25 +[Apr 25, 2026 - 14:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 21 : backward pass time 11.55 +[Apr 25, 2026 - 14:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 21 : optimizer step time 0.97 +[Apr 25, 2026 - 14:53:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 21-------------------- +[Apr 25, 2026 - 14:53:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0114 iter count: 1.0 +[Apr 25, 2026 - 14:53:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[Apr 25, 2026 - 14:53:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:53:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:54:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.820412 38.861275 38.744923 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.784668 67.828125 67.654297 +util 97.500000 99.000000 96.000000 +[Apr 25, 2026 - 14:54:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 22 : avg iteration time 50.13 seconds +[Apr 25, 2026 - 14:54:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 22 : data loading time 0.00 +[Apr 25, 2026 - 14:54:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 22 : avg forward pass time 13.03 +[Apr 25, 2026 - 14:54:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 22 : backward pass time 11.56 +[Apr 25, 2026 - 14:54:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 22 : optimizer step time 0.99 +[Apr 25, 2026 - 14:54:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 22-------------------- +[Apr 25, 2026 - 14:54:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0103 iter count: 1.0 +[Apr 25, 2026 - 14:54:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[Apr 25, 2026 - 14:54:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:54:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:55:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.820468 38.861492 38.744915 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.872559 68.005859 67.828125 +util 95.500000 99.000000 92.000000 +[Apr 25, 2026 - 14:55:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 23 : avg iteration time 49.96 seconds +[Apr 25, 2026 - 14:55:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 23 : data loading time 0.00 +[Apr 25, 2026 - 14:55:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 23 : avg forward pass time 12.94 +[Apr 25, 2026 - 14:55:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 23 : backward pass time 11.55 +[Apr 25, 2026 - 14:55:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 23 : optimizer step time 0.98 +[Apr 25, 2026 - 14:55:31 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 23-------------------- +[Apr 25, 2026 - 14:55:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0119 iter count: 1.0 +[Apr 25, 2026 - 14:55:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 25, 2026 - 14:55:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:55:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:56:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.820986 38.862705 38.745796 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.911621 67.986328 67.886719 +util 96.000000 99.000000 93.000000 +[Apr 25, 2026 - 14:56:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 24 : avg iteration time 50.01 seconds +[Apr 25, 2026 - 14:56:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 24 : data loading time 0.00 +[Apr 25, 2026 - 14:56:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 24 : avg forward pass time 12.96 +[Apr 25, 2026 - 14:56:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 24 : backward pass time 11.54 +[Apr 25, 2026 - 14:56:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 24 : optimizer step time 0.98 +[Apr 25, 2026 - 14:56:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 24-------------------- +[Apr 25, 2026 - 14:56:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 14:56:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 14:56:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:56:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.820711 38.861382 38.745792 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.167480 133.191406 133.095703 +util 68.250000 73.000000 64.000000 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 25 : avg iteration time 169.41 seconds +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 25 : data loading time 0.00 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 25 : avg forward pass time 74.99 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 25 : backward pass time 9.18 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 25 : optimizer step time 0.34 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 25-------------------- +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 18994.6445 iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5939 iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 18994.6445iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.2332iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5834 iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0628 iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 46661.1836iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.2120 iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1363624.8750iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5939iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 1.6603 iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 27.6756 iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 0.5834 iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 3.2120 iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.4782 iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1049.9153iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.5745 iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 9404.8350iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:59:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 14:59:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.820757 38.861599 38.745804 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.882324 67.964844 67.634766 +util 96.000000 100.000000 92.000000 +[Apr 25, 2026 - 14:59:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 26 : avg iteration time 48.49 seconds +[Apr 25, 2026 - 14:59:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 26 : data loading time 0.00 +[Apr 25, 2026 - 14:59:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 26 : avg forward pass time 12.22 +[Apr 25, 2026 - 14:59:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 26 : backward pass time 11.52 +[Apr 25, 2026 - 14:59:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 26 : optimizer step time 0.98 +[Apr 25, 2026 - 14:59:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 26-------------------- +[Apr 25, 2026 - 14:59:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0127 iter count: 1.0 +[Apr 25, 2026 - 14:59:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[Apr 25, 2026 - 14:59:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 14:59:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:00:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.820772 38.861629 38.745804 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.882324 67.986328 67.847656 +util 97.000000 99.000000 95.000000 +[Apr 25, 2026 - 15:00:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 27 : avg iteration time 52.73 seconds +[Apr 25, 2026 - 15:00:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 27 : data loading time 0.00 +[Apr 25, 2026 - 15:00:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 27 : avg forward pass time 14.34 +[Apr 25, 2026 - 15:00:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 27 : backward pass time 11.53 +[Apr 25, 2026 - 15:00:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 27 : optimizer step time 0.98 +[Apr 25, 2026 - 15:00:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 27-------------------- +[Apr 25, 2026 - 15:00:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0108 iter count: 1.0 +[Apr 25, 2026 - 15:00:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 25, 2026 - 15:00:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:00:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:01:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.820761 38.861607 38.745781 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.940918 67.986328 67.925781 +util 96.500000 99.000000 94.000000 +[Apr 25, 2026 - 15:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 28 : avg iteration time 52.79 seconds +[Apr 25, 2026 - 15:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 28 : data loading time 0.00 +[Apr 25, 2026 - 15:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 28 : avg forward pass time 14.38 +[Apr 25, 2026 - 15:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 28 : backward pass time 11.52 +[Apr 25, 2026 - 15:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 28 : optimizer step time 0.98 +[Apr 25, 2026 - 15:01:44 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 28-------------------- +[Apr 25, 2026 - 15:01:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0096 iter count: 1.0 +[Apr 25, 2026 - 15:01:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[Apr 25, 2026 - 15:01:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:01:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:02:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.820760 38.861607 38.745785 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.940918 67.986328 67.925781 +util 96.250000 100.000000 93.000000 +[Apr 25, 2026 - 15:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 29 : avg iteration time 49.88 seconds +[Apr 25, 2026 - 15:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 29 : data loading time 0.00 +[Apr 25, 2026 - 15:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 29 : avg forward pass time 12.90 +[Apr 25, 2026 - 15:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 29 : backward pass time 11.53 +[Apr 25, 2026 - 15:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 29 : optimizer step time 0.99 +[Apr 25, 2026 - 15:02:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 29-------------------- +[Apr 25, 2026 - 15:02:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 25, 2026 - 15:02:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 25, 2026 - 15:02:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:02:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821106 38.862579 38.745975 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.177734 133.191406 133.136719 +util 67.250000 72.000000 63.000000 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 30 : avg iteration time 178.30 seconds +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 30 : data loading time 0.00 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 30 : avg forward pass time 79.44 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 30 : backward pass time 9.15 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 30 : optimizer step time 0.34 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 30-------------------- +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 134435.8940 iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6647 iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 134435.8940iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.8458iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.8820 iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.9650 iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 274551.5938iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 43.1163iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 8285894.5000iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6647iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.1309 iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 28620.4043iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.4920 iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 7974.5210iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.9593 iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 371.9092iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.8492 iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 16292.4287iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:05:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 15:06:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821114 38.862595 38.745960 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.721191 68.103516 67.593750 +util 97.500000 99.000000 96.000000 +[Apr 25, 2026 - 15:06:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 31 : avg iteration time 48.45 seconds +[Apr 25, 2026 - 15:06:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 31 : data loading time 0.00 +[Apr 25, 2026 - 15:06:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 31 : avg forward pass time 12.20 +[Apr 25, 2026 - 15:06:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 31 : backward pass time 11.53 +[Apr 25, 2026 - 15:06:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 31 : optimizer step time 0.98 +[Apr 25, 2026 - 15:06:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 31-------------------- +[Apr 25, 2026 - 15:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0129 iter count: 1.0 +[Apr 25, 2026 - 15:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[Apr 25, 2026 - 15:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:06:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:07:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821124 38.862587 38.745983 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.994629 68.003906 67.966797 +util 95.000000 99.000000 90.000000 +[Apr 25, 2026 - 15:07:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 32 : avg iteration time 49.93 seconds +[Apr 25, 2026 - 15:07:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 32 : data loading time 0.00 +[Apr 25, 2026 - 15:07:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 32 : avg forward pass time 12.94 +[Apr 25, 2026 - 15:07:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 32 : backward pass time 11.54 +[Apr 25, 2026 - 15:07:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 32 : optimizer step time 0.98 +[Apr 25, 2026 - 15:07:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 32-------------------- +[Apr 25, 2026 - 15:07:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 15:07:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 15:07:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:07:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:07:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821127 38.862587 38.745972 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.926270 67.986328 67.906250 +util 99.000000 99.000000 99.000000 +[Apr 25, 2026 - 15:07:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 33 : avg iteration time 47.00 seconds +[Apr 25, 2026 - 15:07:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 33 : data loading time 0.00 +[Apr 25, 2026 - 15:07:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 33 : avg forward pass time 11.47 +[Apr 25, 2026 - 15:07:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 33 : backward pass time 11.54 +[Apr 25, 2026 - 15:07:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 33 : optimizer step time 0.98 +[Apr 25, 2026 - 15:07:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 33-------------------- +[Apr 25, 2026 - 15:07:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0111 iter count: 1.0 +[Apr 25, 2026 - 15:07:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[Apr 25, 2026 - 15:07:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:07:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:08:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821186 38.862846 38.745972 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.853027 67.886719 67.751953 +util 97.000000 100.000000 94.000000 +[Apr 25, 2026 - 15:08:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 34 : avg iteration time 52.85 seconds +[Apr 25, 2026 - 15:08:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 34 : data loading time 0.00 +[Apr 25, 2026 - 15:08:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 34 : avg forward pass time 14.40 +[Apr 25, 2026 - 15:08:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 34 : backward pass time 11.53 +[Apr 25, 2026 - 15:08:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 34 : optimizer step time 0.98 +[Apr 25, 2026 - 15:08:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 34-------------------- +[Apr 25, 2026 - 15:08:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0093 iter count: 1.0 +[Apr 25, 2026 - 15:08:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[Apr 25, 2026 - 15:08:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:08:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.820915 38.861290 38.746082 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.191895 133.210938 133.134766 +util 72.250000 77.000000 68.000000 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 35 : avg iteration time 172.14 seconds +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 35 : data loading time 0.00 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 35 : avg forward pass time 76.23 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 35 : backward pass time 9.12 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 35 : optimizer step time 0.34 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 35-------------------- +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 7085.8385 iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5187 iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 7085.8385iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.5257iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.2027 iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.0491 iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 11202.8018iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 11.0823iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 179558.6406iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5187iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.0110 iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 55.8157 iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 1.2027 iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 11.0823 iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.8858 iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 17529.1387iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.7990 iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 269.8618iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:11:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 15:12:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.820972 38.861534 38.746082 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.638184 67.947266 67.535156 +util 98.250000 99.000000 97.000000 +[Apr 25, 2026 - 15:12:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 36 : avg iteration time 51.30 seconds +[Apr 25, 2026 - 15:12:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 36 : data loading time 0.00 +[Apr 25, 2026 - 15:12:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 36 : avg forward pass time 13.64 +[Apr 25, 2026 - 15:12:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 36 : backward pass time 11.53 +[Apr 25, 2026 - 15:12:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 36 : optimizer step time 0.98 +[Apr 25, 2026 - 15:12:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 36-------------------- +[Apr 25, 2026 - 15:12:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0123 iter count: 1.0 +[Apr 25, 2026 - 15:12:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[Apr 25, 2026 - 15:12:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:12:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:13:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821220 38.862545 38.746059 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.028809 68.162109 67.984375 +util 94.500000 99.000000 90.000000 +[Apr 25, 2026 - 15:13:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 37 : avg iteration time 46.92 seconds +[Apr 25, 2026 - 15:13:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 37 : data loading time 0.00 +[Apr 25, 2026 - 15:13:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 37 : avg forward pass time 11.43 +[Apr 25, 2026 - 15:13:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 37 : backward pass time 11.54 +[Apr 25, 2026 - 15:13:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 37 : optimizer step time 0.97 +[Apr 25, 2026 - 15:13:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 37-------------------- +[Apr 25, 2026 - 15:13:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0113 iter count: 1.0 +[Apr 25, 2026 - 15:13:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[Apr 25, 2026 - 15:13:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:13:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:14:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821342 38.863049 38.746059 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.892090 67.964844 67.673828 +util 95.250000 99.000000 91.000000 +[Apr 25, 2026 - 15:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 38 : avg iteration time 52.82 seconds +[Apr 25, 2026 - 15:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 38 : data loading time 0.00 +[Apr 25, 2026 - 15:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 38 : avg forward pass time 14.39 +[Apr 25, 2026 - 15:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 38 : backward pass time 11.52 +[Apr 25, 2026 - 15:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 38 : optimizer step time 0.98 +[Apr 25, 2026 - 15:14:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 38-------------------- +[Apr 25, 2026 - 15:14:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0108 iter count: 1.0 +[Apr 25, 2026 - 15:14:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 25, 2026 - 15:14:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:14:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:15:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821484 38.863544 38.746109 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.004395 68.123047 67.964844 +util 97.000000 99.000000 91.000000 +[Apr 25, 2026 - 15:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 39 : avg iteration time 49.89 seconds +[Apr 25, 2026 - 15:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 39 : data loading time 0.00 +[Apr 25, 2026 - 15:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 39 : avg forward pass time 12.93 +[Apr 25, 2026 - 15:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 39 : backward pass time 11.53 +[Apr 25, 2026 - 15:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 39 : optimizer step time 0.98 +[Apr 25, 2026 - 15:15:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 39-------------------- +[Apr 25, 2026 - 15:15:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0103 iter count: 1.0 +[Apr 25, 2026 - 15:15:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[Apr 25, 2026 - 15:15:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:15:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.820936 38.861374 38.746094 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.099121 133.113281 133.056641 +util 68.500000 73.000000 64.000000 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 40 : avg iteration time 172.13 seconds +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 40 : data loading time 0.00 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 40 : avg forward pass time 76.37 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 40 : backward pass time 9.14 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 40 : optimizer step time 0.34 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 40-------------------- +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 8467.6315 iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6253 iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 8467.6315iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.8005iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5774 iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 5.9278 iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 12423.3555iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.1731 iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 140870.5469iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6253iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.2852 iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 713.6595iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.5170 iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 1134.6685iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.5568 iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 166.2551iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.9970 iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 2963.3447iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:17:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 15:18:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.820943 38.861401 38.746094 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.979980 67.984375 67.966797 +util 96.500000 100.000000 93.000000 +[Apr 25, 2026 - 15:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 41 : avg iteration time 54.20 seconds +[Apr 25, 2026 - 15:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 41 : data loading time 0.00 +[Apr 25, 2026 - 15:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 41 : avg forward pass time 15.09 +[Apr 25, 2026 - 15:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 41 : backward pass time 11.49 +[Apr 25, 2026 - 15:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 41 : optimizer step time 0.99 +[Apr 25, 2026 - 15:18:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 41-------------------- +[Apr 25, 2026 - 15:18:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0106 iter count: 1.0 +[Apr 25, 2026 - 15:18:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[Apr 25, 2026 - 15:18:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:18:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:19:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821330 38.862968 38.746075 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.970215 67.986328 67.964844 +util 98.250000 99.000000 97.000000 +[Apr 25, 2026 - 15:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 42 : avg iteration time 49.95 seconds +[Apr 25, 2026 - 15:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 42 : data loading time 0.00 +[Apr 25, 2026 - 15:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 42 : avg forward pass time 12.96 +[Apr 25, 2026 - 15:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 42 : backward pass time 11.51 +[Apr 25, 2026 - 15:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 42 : optimizer step time 0.98 +[Apr 25, 2026 - 15:19:40 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 42-------------------- +[Apr 25, 2026 - 15:19:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0116 iter count: 1.0 +[Apr 25, 2026 - 15:19:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[Apr 25, 2026 - 15:19:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:19:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:20:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821348 38.863018 38.746075 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.936035 67.945312 67.908203 +util 98.000000 100.000000 96.000000 +[Apr 25, 2026 - 15:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 43 : avg iteration time 46.99 seconds +[Apr 25, 2026 - 15:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 43 : data loading time 0.00 +[Apr 25, 2026 - 15:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 43 : avg forward pass time 11.48 +[Apr 25, 2026 - 15:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 43 : backward pass time 11.53 +[Apr 25, 2026 - 15:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 43 : optimizer step time 0.98 +[Apr 25, 2026 - 15:20:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 43-------------------- +[Apr 25, 2026 - 15:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 25, 2026 - 15:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 25, 2026 - 15:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:20:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:21:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821407 38.863247 38.746075 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.960449 67.984375 67.888672 +util 98.000000 99.000000 97.000000 +[Apr 25, 2026 - 15:21:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 44 : avg iteration time 49.91 seconds +[Apr 25, 2026 - 15:21:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 44 : data loading time 0.00 +[Apr 25, 2026 - 15:21:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 44 : avg forward pass time 12.93 +[Apr 25, 2026 - 15:21:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 44 : backward pass time 11.54 +[Apr 25, 2026 - 15:21:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 44 : optimizer step time 0.97 +[Apr 25, 2026 - 15:21:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 44-------------------- +[Apr 25, 2026 - 15:21:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0108 iter count: 1.0 +[Apr 25, 2026 - 15:21:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 25, 2026 - 15:21:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:21:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.820991 38.861599 38.746078 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.099121 133.115234 133.093750 +util 71.250000 75.000000 67.000000 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 45 : avg iteration time 172.70 seconds +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 45 : data loading time 0.00 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 45 : avg forward pass time 77.05 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 45 : backward pass time 9.17 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 45 : optimizer step time 0.34 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 45-------------------- +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 52976.3601 iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6710 iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 52976.3601iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4832iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.2806 iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0380 iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 98976.0859iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 95.7042iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1297446.5000iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6710iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.6704 iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 622003.0000iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.1515 iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 546.2162iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.0840 iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 26054.2539iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.1959 iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 32593.1406iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:24:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 15:25:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821011 38.861599 38.746105 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.906738 67.925781 67.849609 +util 98.000000 100.000000 96.000000 +[Apr 25, 2026 - 15:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 46 : avg iteration time 54.49 seconds +[Apr 25, 2026 - 15:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 46 : data loading time 0.00 +[Apr 25, 2026 - 15:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 46 : avg forward pass time 15.22 +[Apr 25, 2026 - 15:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 46 : backward pass time 11.51 +[Apr 25, 2026 - 15:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 46 : optimizer step time 1.00 +[Apr 25, 2026 - 15:25:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 46-------------------- +[Apr 25, 2026 - 15:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0107 iter count: 1.0 +[Apr 25, 2026 - 15:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[Apr 25, 2026 - 15:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:25:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:25:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821173 38.862286 38.746082 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.716309 68.201172 67.554688 +util 94.500000 99.000000 90.000000 +[Apr 25, 2026 - 15:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 47 : avg iteration time 49.94 seconds +[Apr 25, 2026 - 15:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 47 : data loading time 0.00 +[Apr 25, 2026 - 15:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 47 : avg forward pass time 12.95 +[Apr 25, 2026 - 15:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 47 : backward pass time 11.51 +[Apr 25, 2026 - 15:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 47 : optimizer step time 0.99 +[Apr 25, 2026 - 15:25:54 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 47-------------------- +[Apr 25, 2026 - 15:25:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0114 iter count: 1.0 +[Apr 25, 2026 - 15:25:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[Apr 25, 2026 - 15:25:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:25:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:26:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821306 38.862789 38.746098 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.014160 68.162109 67.964844 +util 98.750000 99.000000 98.000000 +[Apr 25, 2026 - 15:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 48 : avg iteration time 52.80 seconds +[Apr 25, 2026 - 15:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 48 : data loading time 0.00 +[Apr 25, 2026 - 15:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 48 : avg forward pass time 14.38 +[Apr 25, 2026 - 15:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 48 : backward pass time 11.52 +[Apr 25, 2026 - 15:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 48 : optimizer step time 0.98 +[Apr 25, 2026 - 15:26:47 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 48-------------------- +[Apr 25, 2026 - 15:26:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0117 iter count: 1.0 +[Apr 25, 2026 - 15:26:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[Apr 25, 2026 - 15:26:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:26:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:27:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821308 38.862778 38.746105 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.940918 67.984375 67.810547 +util 96.000000 99.000000 93.000000 +[Apr 25, 2026 - 15:27:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 49 : avg iteration time 52.83 seconds +[Apr 25, 2026 - 15:27:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 49 : data loading time 0.00 +[Apr 25, 2026 - 15:27:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 49 : avg forward pass time 14.39 +[Apr 25, 2026 - 15:27:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 49 : backward pass time 11.54 +[Apr 25, 2026 - 15:27:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 49 : optimizer step time 0.97 +[Apr 25, 2026 - 15:27:40 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 49-------------------- +[Apr 25, 2026 - 15:27:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0109 iter count: 1.0 +[Apr 25, 2026 - 15:27:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[Apr 25, 2026 - 15:27:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:27:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821032 38.861694 38.746105 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.113770 133.115234 133.113281 +util 72.000000 77.000000 67.000000 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 50 : avg iteration time 169.45 seconds +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 50 : data loading time 0.00 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 50 : avg forward pass time 74.88 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 50 : backward pass time 9.13 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 50 : optimizer step time 0.34 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 50-------------------- +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 65795.0306 iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6998 iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 65795.0306iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3605iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.6780 iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.2960 iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 123534.5000iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 28.6721iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2173879.2500iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6998iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.3528 iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 6037.1338iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.1787 iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 4261.3521iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.0923 iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 65.6697 iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.0019 iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 2992.1172iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:30:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 15:31:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821035 38.861710 38.746105 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.979980 68.142578 67.925781 +util 96.750000 100.000000 94.000000 +[Apr 25, 2026 - 15:31:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 51 : avg iteration time 51.36 seconds +[Apr 25, 2026 - 15:31:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 51 : data loading time 0.00 +[Apr 25, 2026 - 15:31:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 51 : avg forward pass time 13.67 +[Apr 25, 2026 - 15:31:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 51 : backward pass time 11.52 +[Apr 25, 2026 - 15:31:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 51 : optimizer step time 0.98 +[Apr 25, 2026 - 15:31:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 51-------------------- +[Apr 25, 2026 - 15:31:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0103 iter count: 1.0 +[Apr 25, 2026 - 15:31:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[Apr 25, 2026 - 15:31:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:31:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:32:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821158 38.862190 38.746105 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.023926 68.083984 68.003906 +util 96.750000 99.000000 94.000000 +[Apr 25, 2026 - 15:32:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 52 : avg iteration time 49.91 seconds +[Apr 25, 2026 - 15:32:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 52 : data loading time 0.00 +[Apr 25, 2026 - 15:32:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 52 : avg forward pass time 12.94 +[Apr 25, 2026 - 15:32:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 52 : backward pass time 11.53 +[Apr 25, 2026 - 15:32:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 52 : optimizer step time 0.97 +[Apr 25, 2026 - 15:32:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 52-------------------- +[Apr 25, 2026 - 15:32:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0122 iter count: 1.0 +[Apr 25, 2026 - 15:32:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[Apr 25, 2026 - 15:32:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:32:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:33:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821255 38.862446 38.746105 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.014160 68.162109 67.964844 +util 98.250000 99.000000 97.000000 +[Apr 25, 2026 - 15:33:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 53 : avg iteration time 50.03 seconds +[Apr 25, 2026 - 15:33:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 53 : data loading time 0.00 +[Apr 25, 2026 - 15:33:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 53 : avg forward pass time 13.00 +[Apr 25, 2026 - 15:33:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 53 : backward pass time 11.54 +[Apr 25, 2026 - 15:33:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 53 : optimizer step time 0.98 +[Apr 25, 2026 - 15:33:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 53-------------------- +[Apr 25, 2026 - 15:33:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 15:33:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 15:33:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:33:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:33:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821259 38.862446 38.746109 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.004395 68.123047 67.964844 +util 96.750000 100.000000 94.000000 +[Apr 25, 2026 - 15:33:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 54 : avg iteration time 49.86 seconds +[Apr 25, 2026 - 15:33:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 54 : data loading time 0.00 +[Apr 25, 2026 - 15:33:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 54 : avg forward pass time 12.90 +[Apr 25, 2026 - 15:33:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 54 : backward pass time 11.52 +[Apr 25, 2026 - 15:33:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 54 : optimizer step time 0.99 +[Apr 25, 2026 - 15:33:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 54-------------------- +[Apr 25, 2026 - 15:33:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0118 iter count: 1.0 +[Apr 25, 2026 - 15:33:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[Apr 25, 2026 - 15:33:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:33:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821103 38.861542 38.746105 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.123535 133.152344 133.037109 +util 71.000000 75.000000 67.000000 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 55 : avg iteration time 172.93 seconds +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 55 : data loading time 0.00 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 55 : avg forward pass time 77.20 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 55 : backward pass time 9.13 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 55 : optimizer step time 0.34 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 55-------------------- +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 18064.2321 iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6133 iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 18064.2321iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.7432iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.0227 iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.3958 iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 32471.9980iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 7.7317 iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 359218.1250iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6133iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.6194 iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 10289.0576iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 1.9859 iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 53.0769 iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.2261 iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 634.0570iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.8857 iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 320.9964iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:36:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 15:37:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821159 38.861797 38.746113 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.877441 68.142578 67.789062 +util 96.000000 99.000000 93.000000 +[Apr 25, 2026 - 15:37:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 56 : avg iteration time 48.45 seconds +[Apr 25, 2026 - 15:37:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 56 : data loading time 0.00 +[Apr 25, 2026 - 15:37:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 56 : avg forward pass time 12.21 +[Apr 25, 2026 - 15:37:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 56 : backward pass time 11.52 +[Apr 25, 2026 - 15:37:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 56 : optimizer step time 0.98 +[Apr 25, 2026 - 15:37:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 56-------------------- +[Apr 25, 2026 - 15:37:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0118 iter count: 1.0 +[Apr 25, 2026 - 15:37:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[Apr 25, 2026 - 15:37:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:37:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:38:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821280 38.862289 38.746105 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.916504 68.181641 67.828125 +util 96.500000 99.000000 94.000000 +[Apr 25, 2026 - 15:38:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 57 : avg iteration time 52.81 seconds +[Apr 25, 2026 - 15:38:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 57 : data loading time 0.00 +[Apr 25, 2026 - 15:38:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 57 : avg forward pass time 14.40 +[Apr 25, 2026 - 15:38:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 57 : backward pass time 11.52 +[Apr 25, 2026 - 15:38:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 57 : optimizer step time 0.98 +[Apr 25, 2026 - 15:38:24 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 57-------------------- +[Apr 25, 2026 - 15:38:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0118 iter count: 1.0 +[Apr 25, 2026 - 15:38:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[Apr 25, 2026 - 15:38:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:38:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:39:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821305 38.862324 38.746136 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.789551 68.201172 67.652344 +util 98.000000 99.000000 97.000000 +[Apr 25, 2026 - 15:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 58 : avg iteration time 49.84 seconds +[Apr 25, 2026 - 15:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 58 : data loading time 0.00 +[Apr 25, 2026 - 15:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 58 : avg forward pass time 12.90 +[Apr 25, 2026 - 15:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 58 : backward pass time 11.53 +[Apr 25, 2026 - 15:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 58 : optimizer step time 0.98 +[Apr 25, 2026 - 15:39:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 58-------------------- +[Apr 25, 2026 - 15:39:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0110 iter count: 1.0 +[Apr 25, 2026 - 15:39:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[Apr 25, 2026 - 15:39:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:39:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:40:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821354 38.862545 38.746124 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.921387 68.201172 67.828125 +util 95.500000 99.000000 91.000000 +[Apr 25, 2026 - 15:40:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 59 : avg iteration time 49.84 seconds +[Apr 25, 2026 - 15:40:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 59 : data loading time 0.00 +[Apr 25, 2026 - 15:40:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 59 : avg forward pass time 12.90 +[Apr 25, 2026 - 15:40:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 59 : backward pass time 11.52 +[Apr 25, 2026 - 15:40:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 59 : optimizer step time 0.99 +[Apr 25, 2026 - 15:40:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 59-------------------- +[Apr 25, 2026 - 15:40:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 15:40:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 25, 2026 - 15:40:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:40:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821115 38.861614 38.746132 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.128418 133.132812 133.115234 +util 68.250000 73.000000 64.000000 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 60 : avg iteration time 172.32 seconds +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 60 : data loading time 0.00 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 60 : avg forward pass time 76.47 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 60 : backward pass time 9.13 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 60 : optimizer step time 0.34 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 60-------------------- +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 16790.8995 iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5723 iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 16790.8995iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.8540iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.7691 iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.2110 iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 28740.3711iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 34.4036iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 248188.0000iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5723iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.6320 iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1427.9023iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.3250 iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 5710.5854iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 1.9414 iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 48.5637 iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.4818 iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1057.4882iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:42:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 15:43:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821142 38.861732 38.746109 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.970215 68.062500 67.693359 +util 96.750000 99.000000 91.000000 +[Apr 25, 2026 - 15:43:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 61 : avg iteration time 54.26 seconds +[Apr 25, 2026 - 15:43:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 61 : data loading time 0.00 +[Apr 25, 2026 - 15:43:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 61 : avg forward pass time 15.10 +[Apr 25, 2026 - 15:43:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 61 : backward pass time 11.54 +[Apr 25, 2026 - 15:43:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 61 : optimizer step time 0.97 +[Apr 25, 2026 - 15:43:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 61-------------------- +[Apr 25, 2026 - 15:43:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0139 iter count: 1.0 +[Apr 25, 2026 - 15:43:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[Apr 25, 2026 - 15:43:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:43:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:44:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821265 38.862236 38.746109 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.141113 68.160156 68.083984 +util 98.250000 99.000000 97.000000 +[Apr 25, 2026 - 15:44:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 62 : avg iteration time 49.89 seconds +[Apr 25, 2026 - 15:44:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 62 : data loading time 0.00 +[Apr 25, 2026 - 15:44:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 62 : avg forward pass time 12.92 +[Apr 25, 2026 - 15:44:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 62 : backward pass time 11.54 +[Apr 25, 2026 - 15:44:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 62 : optimizer step time 0.99 +[Apr 25, 2026 - 15:44:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 62-------------------- +[Apr 25, 2026 - 15:44:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0134 iter count: 1.0 +[Apr 25, 2026 - 15:44:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[Apr 25, 2026 - 15:44:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:44:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:45:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821278 38.862240 38.746132 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.062988 68.082031 68.005859 +util 96.500000 100.000000 93.000000 +[Apr 25, 2026 - 15:45:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 63 : avg iteration time 46.95 seconds +[Apr 25, 2026 - 15:45:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 63 : data loading time 0.00 +[Apr 25, 2026 - 15:45:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 63 : avg forward pass time 11.45 +[Apr 25, 2026 - 15:45:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 63 : backward pass time 11.53 +[Apr 25, 2026 - 15:45:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 63 : optimizer step time 0.99 +[Apr 25, 2026 - 15:45:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 63-------------------- +[Apr 25, 2026 - 15:45:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0119 iter count: 1.0 +[Apr 25, 2026 - 15:45:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 25, 2026 - 15:45:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:45:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:46:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821267 38.862244 38.746109 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.131348 68.179688 67.986328 +util 96.000000 99.000000 93.000000 +[Apr 25, 2026 - 15:46:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 64 : avg iteration time 49.89 seconds +[Apr 25, 2026 - 15:46:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 64 : data loading time 0.00 +[Apr 25, 2026 - 15:46:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 64 : avg forward pass time 12.94 +[Apr 25, 2026 - 15:46:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 64 : backward pass time 11.51 +[Apr 25, 2026 - 15:46:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 64 : optimizer step time 0.98 +[Apr 25, 2026 - 15:46:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 64-------------------- +[Apr 25, 2026 - 15:46:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0123 iter count: 1.0 +[Apr 25, 2026 - 15:46:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[Apr 25, 2026 - 15:46:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:46:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821359 38.862591 38.746109 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.123535 133.132812 133.095703 +util 71.250000 76.000000 67.000000 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 65 : avg iteration time 175.25 seconds +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 65 : data loading time 0.00 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 65 : avg forward pass time 77.75 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 65 : backward pass time 9.16 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 65 : optimizer step time 0.34 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 65-------------------- +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 59853.0330 iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6321 iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 59853.0330iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.9630iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7554 iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0893 iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 99181.2422iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.5299 iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1437880.1250iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6321iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.3425 iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 800.3698iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.3789 iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 6359.9897iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.3388 iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 794.4427iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.6231 iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1402.8640iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 15:50:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821364 38.862595 38.746109 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.994629 68.142578 67.945312 +util 96.000000 99.000000 92.000000 +[Apr 25, 2026 - 15:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 66 : avg iteration time 51.45 seconds +[Apr 25, 2026 - 15:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 66 : data loading time 0.00 +[Apr 25, 2026 - 15:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 66 : avg forward pass time 13.68 +[Apr 25, 2026 - 15:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 66 : backward pass time 11.53 +[Apr 25, 2026 - 15:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 66 : optimizer step time 0.99 +[Apr 25, 2026 - 15:50:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 66-------------------- +[Apr 25, 2026 - 15:50:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0139 iter count: 1.0 +[Apr 25, 2026 - 15:50:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[Apr 25, 2026 - 15:50:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:50:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:50:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821371 38.862595 38.746155 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.809082 67.828125 67.751953 +util 97.250000 100.000000 92.000000 +[Apr 25, 2026 - 15:50:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 67 : avg iteration time 49.87 seconds +[Apr 25, 2026 - 15:50:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 67 : data loading time 0.00 +[Apr 25, 2026 - 15:50:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 67 : avg forward pass time 12.92 +[Apr 25, 2026 - 15:50:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 67 : backward pass time 11.53 +[Apr 25, 2026 - 15:50:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 67 : optimizer step time 0.98 +[Apr 25, 2026 - 15:50:54 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 67-------------------- +[Apr 25, 2026 - 15:50:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 15:50:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 15:50:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:50:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:51:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821277 38.862225 38.746132 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.921387 67.964844 67.791016 +util 95.000000 100.000000 91.000000 +[Apr 25, 2026 - 15:51:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 68 : avg iteration time 52.81 seconds +[Apr 25, 2026 - 15:51:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 68 : data loading time 0.00 +[Apr 25, 2026 - 15:51:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 68 : avg forward pass time 14.37 +[Apr 25, 2026 - 15:51:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 68 : backward pass time 11.54 +[Apr 25, 2026 - 15:51:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 68 : optimizer step time 0.97 +[Apr 25, 2026 - 15:51:47 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 68-------------------- +[Apr 25, 2026 - 15:51:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0124 iter count: 1.0 +[Apr 25, 2026 - 15:51:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[Apr 25, 2026 - 15:51:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:51:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:52:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821279 38.862247 38.746132 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.921387 67.964844 67.791016 +util 95.000000 99.000000 91.000000 +[Apr 25, 2026 - 15:52:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 69 : avg iteration time 52.80 seconds +[Apr 25, 2026 - 15:52:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 69 : data loading time 0.00 +[Apr 25, 2026 - 15:52:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 69 : avg forward pass time 14.37 +[Apr 25, 2026 - 15:52:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 69 : backward pass time 11.54 +[Apr 25, 2026 - 15:52:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 69 : optimizer step time 0.98 +[Apr 25, 2026 - 15:52:40 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 69-------------------- +[Apr 25, 2026 - 15:52:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 25, 2026 - 15:52:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 25, 2026 - 15:52:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:52:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.820996 38.860970 38.746162 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.133789 133.136719 133.132812 +util 69.000000 74.000000 64.000000 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 70 : avg iteration time 172.04 seconds +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 70 : data loading time 0.00 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 70 : avg forward pass time 76.17 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 70 : backward pass time 9.12 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 70 : optimizer step time 0.34 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 70-------------------- +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 44438.6785 iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6323 iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 44438.6785iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.0209iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.9840 iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.1859 iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 86355.0859iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 7.1559 iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1744225.8750iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6323iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.9082 iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 335.7780iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.8337 iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2137.3503iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 1.6802 iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 28.8011 iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.9390 iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 2638.4248iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:55:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 15:56:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821066 38.861225 38.746185 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.804199 67.847656 67.673828 +util 98.250000 99.000000 97.000000 +[Apr 25, 2026 - 15:56:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 71 : avg iteration time 51.24 seconds +[Apr 25, 2026 - 15:56:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 71 : data loading time 0.00 +[Apr 25, 2026 - 15:56:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 71 : avg forward pass time 13.59 +[Apr 25, 2026 - 15:56:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 71 : backward pass time 11.53 +[Apr 25, 2026 - 15:56:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 71 : optimizer step time 0.98 +[Apr 25, 2026 - 15:56:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 71-------------------- +[Apr 25, 2026 - 15:56:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0143 iter count: 1.0 +[Apr 25, 2026 - 15:56:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0143 iter count: 1.0 +[Apr 25, 2026 - 15:56:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:56:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:57:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821378 38.862469 38.746162 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.657715 67.673828 67.652344 +util 98.250000 100.000000 96.000000 +[Apr 25, 2026 - 15:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 72 : avg iteration time 49.90 seconds +[Apr 25, 2026 - 15:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 72 : data loading time 0.00 +[Apr 25, 2026 - 15:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 72 : avg forward pass time 12.93 +[Apr 25, 2026 - 15:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 72 : backward pass time 11.52 +[Apr 25, 2026 - 15:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 72 : optimizer step time 0.97 +[Apr 25, 2026 - 15:57:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 72-------------------- +[Apr 25, 2026 - 15:57:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0130 iter count: 1.0 +[Apr 25, 2026 - 15:57:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[Apr 25, 2026 - 15:57:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:57:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:58:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821433 38.862717 38.746140 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.848145 68.025391 67.789062 +util 97.000000 100.000000 94.000000 +[Apr 25, 2026 - 15:58:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 73 : avg iteration time 52.77 seconds +[Apr 25, 2026 - 15:58:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 73 : data loading time 0.00 +[Apr 25, 2026 - 15:58:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 73 : avg forward pass time 14.36 +[Apr 25, 2026 - 15:58:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 73 : backward pass time 11.52 +[Apr 25, 2026 - 15:58:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 73 : optimizer step time 0.99 +[Apr 25, 2026 - 15:58:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 73-------------------- +[Apr 25, 2026 - 15:58:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 25, 2026 - 15:58:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 25, 2026 - 15:58:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:58:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 15:58:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821445 38.862740 38.746140 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.794434 67.847656 67.634766 +util 94.750000 99.000000 90.000000 +[Apr 25, 2026 - 15:58:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 74 : avg iteration time 52.85 seconds +[Apr 25, 2026 - 15:58:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 74 : data loading time 0.00 +[Apr 25, 2026 - 15:58:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 74 : avg forward pass time 14.40 +[Apr 25, 2026 - 15:58:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 74 : backward pass time 11.53 +[Apr 25, 2026 - 15:58:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 74 : optimizer step time 0.98 +[Apr 25, 2026 - 15:58:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 74-------------------- +[Apr 25, 2026 - 15:58:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0130 iter count: 1.0 +[Apr 25, 2026 - 15:58:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[Apr 25, 2026 - 15:58:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 15:58:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821642 38.862835 38.746338 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.084961 133.117188 133.074219 +util 71.250000 76.000000 67.000000 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 75 : avg iteration time 172.27 seconds +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 75 : data loading time 0.00 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 75 : avg forward pass time 76.43 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 75 : backward pass time 9.13 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 75 : optimizer step time 0.34 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 75-------------------- +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 102423.7351 iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4695 iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 102423.7351iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3083iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.1878 iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.4966 iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 235978.6562iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 10.7570iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 3247025.5000iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4695iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.9100 iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 18399.0625iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.6356 iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 1438.2784iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.6201 iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 188.7025iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.3606 iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 6131.7021iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:01:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 16:02:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.821709 38.863079 38.746338 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.911621 67.984375 67.693359 +util 97.000000 99.000000 91.000000 +[Apr 25, 2026 - 16:02:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 76 : avg iteration time 54.23 seconds +[Apr 25, 2026 - 16:02:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 76 : data loading time 0.00 +[Apr 25, 2026 - 16:02:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 76 : avg forward pass time 15.09 +[Apr 25, 2026 - 16:02:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 76 : backward pass time 11.54 +[Apr 25, 2026 - 16:02:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 76 : optimizer step time 0.97 +[Apr 25, 2026 - 16:02:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 76-------------------- +[Apr 25, 2026 - 16:02:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0131 iter count: 1.0 +[Apr 25, 2026 - 16:02:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0131 iter count: 1.0 +[Apr 25, 2026 - 16:02:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:02:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:03:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.822135 38.864796 38.746338 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.916504 67.984375 67.712891 +util 98.000000 99.000000 97.000000 +[Apr 25, 2026 - 16:03:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 77 : avg iteration time 52.78 seconds +[Apr 25, 2026 - 16:03:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 77 : data loading time 0.00 +[Apr 25, 2026 - 16:03:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 77 : avg forward pass time 14.37 +[Apr 25, 2026 - 16:03:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 77 : backward pass time 11.53 +[Apr 25, 2026 - 16:03:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 77 : optimizer step time 0.98 +[Apr 25, 2026 - 16:03:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 77-------------------- +[Apr 25, 2026 - 16:03:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0141 iter count: 1.0 +[Apr 25, 2026 - 16:03:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0141 iter count: 1.0 +[Apr 25, 2026 - 16:03:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:03:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:04:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.822129 38.864796 38.746338 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.892090 67.945312 67.732422 +util 96.750000 99.000000 90.000000 +[Apr 25, 2026 - 16:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 78 : avg iteration time 52.81 seconds +[Apr 25, 2026 - 16:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 78 : data loading time 0.00 +[Apr 25, 2026 - 16:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 78 : avg forward pass time 14.39 +[Apr 25, 2026 - 16:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 78 : backward pass time 11.52 +[Apr 25, 2026 - 16:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 78 : optimizer step time 0.98 +[Apr 25, 2026 - 16:04:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 78-------------------- +[Apr 25, 2026 - 16:04:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0127 iter count: 1.0 +[Apr 25, 2026 - 16:04:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[Apr 25, 2026 - 16:04:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:04:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:05:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.822041 38.864429 38.746342 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.033691 68.123047 68.003906 +util 97.000000 100.000000 94.000000 +[Apr 25, 2026 - 16:05:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 79 : avg iteration time 52.76 seconds +[Apr 25, 2026 - 16:05:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 79 : data loading time 0.00 +[Apr 25, 2026 - 16:05:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 79 : avg forward pass time 14.36 +[Apr 25, 2026 - 16:05:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 79 : backward pass time 11.54 +[Apr 25, 2026 - 16:05:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 79 : optimizer step time 0.98 +[Apr 25, 2026 - 16:05:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 79-------------------- +[Apr 25, 2026 - 16:05:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0147 iter count: 1.0 +[Apr 25, 2026 - 16:05:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0147 iter count: 1.0 +[Apr 25, 2026 - 16:05:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:05:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823034 38.866249 38.747150 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.140137 133.154297 133.097656 +util 69.750000 76.000000 66.000000 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 80 : avg iteration time 167.24 seconds +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 80 : data loading time 0.00 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 80 : avg forward pass time 74.35 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 80 : backward pass time 9.14 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 80 : optimizer step time 0.34 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 80-------------------- +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 103874.0187 iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5741 iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 103874.0187iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.6642iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.9479 iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.3060 iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 145495.3750iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 6.6576 iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2217827.2500iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5741iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 0.9559 iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 6.7653 iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 7.3060 iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2217827.2500iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.8463 iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 16198.2646iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.2010 iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 81.6110 iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:08:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 16:08:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823026 38.866249 38.747131 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.945801 67.984375 67.830078 +util 98.000000 99.000000 97.000000 +[Apr 25, 2026 - 16:08:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 81 : avg iteration time 48.38 seconds +[Apr 25, 2026 - 16:08:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 81 : data loading time 0.00 +[Apr 25, 2026 - 16:08:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 81 : avg forward pass time 12.17 +[Apr 25, 2026 - 16:08:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 81 : backward pass time 11.51 +[Apr 25, 2026 - 16:08:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 81 : optimizer step time 0.99 +[Apr 25, 2026 - 16:08:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 81-------------------- +[Apr 25, 2026 - 16:08:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0148 iter count: 1.0 +[Apr 25, 2026 - 16:08:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0148 iter count: 1.0 +[Apr 25, 2026 - 16:08:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:08:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:09:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823093 38.866520 38.747131 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.965332 67.966797 67.964844 +util 95.250000 99.000000 91.000000 +[Apr 25, 2026 - 16:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 82 : avg iteration time 46.94 seconds +[Apr 25, 2026 - 16:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 82 : data loading time 0.00 +[Apr 25, 2026 - 16:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 82 : avg forward pass time 11.44 +[Apr 25, 2026 - 16:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 82 : backward pass time 11.55 +[Apr 25, 2026 - 16:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 82 : optimizer step time 0.97 +[Apr 25, 2026 - 16:09:46 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 82-------------------- +[Apr 25, 2026 - 16:09:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0146 iter count: 1.0 +[Apr 25, 2026 - 16:09:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0146 iter count: 1.0 +[Apr 25, 2026 - 16:09:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:09:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:10:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823094 38.866497 38.747131 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.931152 68.064453 67.886719 +util 97.000000 99.000000 95.000000 +[Apr 25, 2026 - 16:10:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 83 : avg iteration time 49.84 seconds +[Apr 25, 2026 - 16:10:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 83 : data loading time 0.00 +[Apr 25, 2026 - 16:10:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 83 : avg forward pass time 12.90 +[Apr 25, 2026 - 16:10:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 83 : backward pass time 11.53 +[Apr 25, 2026 - 16:10:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 83 : optimizer step time 0.98 +[Apr 25, 2026 - 16:10:36 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 83-------------------- +[Apr 25, 2026 - 16:10:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0128 iter count: 1.0 +[Apr 25, 2026 - 16:10:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[Apr 25, 2026 - 16:10:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:10:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:11:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823174 38.866817 38.747131 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.004395 68.123047 67.964844 +util 95.500000 99.000000 91.000000 +[Apr 25, 2026 - 16:11:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 84 : avg iteration time 49.87 seconds +[Apr 25, 2026 - 16:11:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 84 : data loading time 0.00 +[Apr 25, 2026 - 16:11:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 84 : avg forward pass time 12.91 +[Apr 25, 2026 - 16:11:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 84 : backward pass time 11.53 +[Apr 25, 2026 - 16:11:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 84 : optimizer step time 0.99 +[Apr 25, 2026 - 16:11:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 84-------------------- +[Apr 25, 2026 - 16:11:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0147 iter count: 1.0 +[Apr 25, 2026 - 16:11:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0147 iter count: 1.0 +[Apr 25, 2026 - 16:11:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:11:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823254 38.867123 38.747135 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.163086 133.171875 133.136719 +util 72.250000 80.000000 64.000000 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 85 : avg iteration time 172.20 seconds +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 85 : data loading time 0.00 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 85 : avg forward pass time 76.41 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 85 : backward pass time 9.13 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 85 : optimizer step time 0.34 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 85-------------------- +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 6935.8386 iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5733 iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 6935.8386iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.7289iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.6168 iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 5.9079 iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 14883.4336iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 25.3729iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 135361.1250iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5733iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.4926 iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 146.2416iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.4552 iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 135.6826iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.5946 iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 179.3222iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.9079 iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 135361.1250iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0156 iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:14:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 16:15:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823256 38.867123 38.747131 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.975098 68.064453 67.945312 +util 96.500000 99.000000 94.000000 +[Apr 25, 2026 - 16:15:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 86 : avg iteration time 51.32 seconds +[Apr 25, 2026 - 16:15:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 86 : data loading time 0.00 +[Apr 25, 2026 - 16:15:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 86 : avg forward pass time 13.63 +[Apr 25, 2026 - 16:15:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 86 : backward pass time 11.53 +[Apr 25, 2026 - 16:15:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 86 : optimizer step time 0.99 +[Apr 25, 2026 - 16:15:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 86-------------------- +[Apr 25, 2026 - 16:15:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0139 iter count: 1.0 +[Apr 25, 2026 - 16:15:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[Apr 25, 2026 - 16:15:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:15:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:15:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823252 38.867123 38.747135 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.887207 67.947266 67.867188 +util 96.000000 100.000000 92.000000 +[Apr 25, 2026 - 16:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 87 : avg iteration time 49.86 seconds +[Apr 25, 2026 - 16:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 87 : data loading time 0.00 +[Apr 25, 2026 - 16:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 87 : avg forward pass time 12.90 +[Apr 25, 2026 - 16:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 87 : backward pass time 11.54 +[Apr 25, 2026 - 16:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 87 : optimizer step time 0.98 +[Apr 25, 2026 - 16:15:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 87-------------------- +[Apr 25, 2026 - 16:15:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0149 iter count: 1.0 +[Apr 25, 2026 - 16:15:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0149 iter count: 1.0 +[Apr 25, 2026 - 16:15:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:15:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:16:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823252 38.867123 38.747131 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.853027 67.927734 67.828125 +util 98.250000 100.000000 96.000000 +[Apr 25, 2026 - 16:16:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 88 : avg iteration time 49.86 seconds +[Apr 25, 2026 - 16:16:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 88 : data loading time 0.00 +[Apr 25, 2026 - 16:16:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 88 : avg forward pass time 12.91 +[Apr 25, 2026 - 16:16:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 88 : backward pass time 11.54 +[Apr 25, 2026 - 16:16:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 88 : optimizer step time 0.97 +[Apr 25, 2026 - 16:16:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 88-------------------- +[Apr 25, 2026 - 16:16:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0152 iter count: 1.0 +[Apr 25, 2026 - 16:16:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0152 iter count: 1.0 +[Apr 25, 2026 - 16:16:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:16:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:17:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823314 38.867374 38.747131 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.867676 67.927734 67.847656 +util 97.500000 100.000000 95.000000 +[Apr 25, 2026 - 16:17:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 89 : avg iteration time 49.88 seconds +[Apr 25, 2026 - 16:17:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 89 : data loading time 0.00 +[Apr 25, 2026 - 16:17:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 89 : avg forward pass time 12.92 +[Apr 25, 2026 - 16:17:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 89 : backward pass time 11.53 +[Apr 25, 2026 - 16:17:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 89 : optimizer step time 0.98 +[Apr 25, 2026 - 16:17:39 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 89-------------------- +[Apr 25, 2026 - 16:17:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0153 iter count: 1.0 +[Apr 25, 2026 - 16:17:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0153 iter count: 1.0 +[Apr 25, 2026 - 16:17:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:17:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823190 38.866772 38.747185 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.144531 133.154297 133.115234 +util 68.750000 76.000000 61.000000 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 90 : avg iteration time 175.23 seconds +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 90 : data loading time 0.00 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 90 : avg forward pass time 78.14 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 90 : backward pass time 9.13 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 90 : optimizer step time 0.34 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 90-------------------- +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 23064.1823 iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6287 iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 23064.1823iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.6983iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.8698 iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.9167 iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 48584.7188iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 5.6948 iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1018124.9375iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6287iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.0114 iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 412.7187iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.6594 iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 82354.9844iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.9435 iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 360.3300iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.9970 iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 401.0152iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:20:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 16:21:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823248 38.867023 38.747185 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.916504 68.064453 67.867188 +util 97.500000 100.000000 95.000000 +[Apr 25, 2026 - 16:21:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 91 : avg iteration time 48.36 seconds +[Apr 25, 2026 - 16:21:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 91 : data loading time 0.00 +[Apr 25, 2026 - 16:21:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 91 : avg forward pass time 12.18 +[Apr 25, 2026 - 16:21:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 91 : backward pass time 11.52 +[Apr 25, 2026 - 16:21:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 91 : optimizer step time 0.97 +[Apr 25, 2026 - 16:21:22 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 91-------------------- +[Apr 25, 2026 - 16:21:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0163 iter count: 1.0 +[Apr 25, 2026 - 16:21:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0163 iter count: 1.0 +[Apr 25, 2026 - 16:21:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:21:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:22:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823247 38.867016 38.747185 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.716309 68.025391 67.613281 +util 97.500000 100.000000 95.000000 +[Apr 25, 2026 - 16:22:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 92 : avg iteration time 52.80 seconds +[Apr 25, 2026 - 16:22:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 92 : data loading time 0.00 +[Apr 25, 2026 - 16:22:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 92 : avg forward pass time 14.38 +[Apr 25, 2026 - 16:22:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 92 : backward pass time 11.53 +[Apr 25, 2026 - 16:22:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 92 : optimizer step time 0.98 +[Apr 25, 2026 - 16:22:15 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 92-------------------- +[Apr 25, 2026 - 16:22:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0139 iter count: 1.0 +[Apr 25, 2026 - 16:22:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[Apr 25, 2026 - 16:22:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:22:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:23:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823245 38.867012 38.747185 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.799316 68.181641 67.671875 +util 96.250000 99.000000 93.000000 +[Apr 25, 2026 - 16:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 93 : avg iteration time 49.78 seconds +[Apr 25, 2026 - 16:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 93 : data loading time 0.00 +[Apr 25, 2026 - 16:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 93 : avg forward pass time 12.87 +[Apr 25, 2026 - 16:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 93 : backward pass time 11.51 +[Apr 25, 2026 - 16:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 93 : optimizer step time 0.99 +[Apr 25, 2026 - 16:23:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 93-------------------- +[Apr 25, 2026 - 16:23:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0145 iter count: 1.0 +[Apr 25, 2026 - 16:23:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0145 iter count: 1.0 +[Apr 25, 2026 - 16:23:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:23:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:23:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823267 38.867031 38.747208 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.936035 68.025391 67.906250 +util 96.500000 99.000000 94.000000 +[Apr 25, 2026 - 16:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 94 : avg iteration time 47.01 seconds +[Apr 25, 2026 - 16:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 94 : data loading time 0.00 +[Apr 25, 2026 - 16:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 94 : avg forward pass time 11.47 +[Apr 25, 2026 - 16:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 94 : backward pass time 11.56 +[Apr 25, 2026 - 16:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 94 : optimizer step time 0.97 +[Apr 25, 2026 - 16:23:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 94-------------------- +[Apr 25, 2026 - 16:23:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0155 iter count: 1.0 +[Apr 25, 2026 - 16:23:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0155 iter count: 1.0 +[Apr 25, 2026 - 16:23:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:23:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823385 38.867390 38.747185 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.162598 133.191406 133.076172 +util 69.250000 78.000000 61.000000 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 95 : avg iteration time 169.16 seconds +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 95 : data loading time 0.00 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 95 : avg forward pass time 74.73 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 95 : backward pass time 9.11 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 95 : optimizer step time 0.34 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 95-------------------- +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 2403131.5338 iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.7057 iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 2403131.5338iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.6215iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.2878 iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 9.1381 iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 2712570.5000iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 13.1384iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 86540176.0000iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.7057iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.1882 iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 79.5552 iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.7725 iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 255.9522iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.0197 iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 3100.7939iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.5393 iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1186.2318iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0163 iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:26:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 16:27:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823386 38.867378 38.747208 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.853027 67.906250 67.693359 +util 96.500000 99.000000 91.000000 +[Apr 25, 2026 - 16:27:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 96 : avg iteration time 48.40 seconds +[Apr 25, 2026 - 16:27:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 96 : data loading time 0.00 +[Apr 25, 2026 - 16:27:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 96 : avg forward pass time 12.18 +[Apr 25, 2026 - 16:27:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 96 : backward pass time 11.53 +[Apr 25, 2026 - 16:27:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 96 : optimizer step time 0.98 +[Apr 25, 2026 - 16:27:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 96-------------------- +[Apr 25, 2026 - 16:27:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0156 iter count: 1.0 +[Apr 25, 2026 - 16:27:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0156 iter count: 1.0 +[Apr 25, 2026 - 16:27:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:27:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:28:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823386 38.867401 38.747185 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.994629 68.142578 67.945312 +util 97.000000 100.000000 94.000000 +[Apr 25, 2026 - 16:28:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 97 : avg iteration time 49.85 seconds +[Apr 25, 2026 - 16:28:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 97 : data loading time 0.00 +[Apr 25, 2026 - 16:28:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 97 : avg forward pass time 12.90 +[Apr 25, 2026 - 16:28:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 97 : backward pass time 11.54 +[Apr 25, 2026 - 16:28:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 97 : optimizer step time 0.97 +[Apr 25, 2026 - 16:28:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 97-------------------- +[Apr 25, 2026 - 16:28:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0172 iter count: 1.0 +[Apr 25, 2026 - 16:28:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0172 iter count: 1.0 +[Apr 25, 2026 - 16:28:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:28:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:29:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823385 38.867378 38.747208 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.896973 67.964844 67.693359 +util 96.750000 100.000000 93.000000 +[Apr 25, 2026 - 16:29:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 98 : avg iteration time 49.90 seconds +[Apr 25, 2026 - 16:29:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 98 : data loading time 0.00 +[Apr 25, 2026 - 16:29:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 98 : avg forward pass time 12.92 +[Apr 25, 2026 - 16:29:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 98 : backward pass time 11.53 +[Apr 25, 2026 - 16:29:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 98 : optimizer step time 0.99 +[Apr 25, 2026 - 16:29:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 98-------------------- +[Apr 25, 2026 - 16:29:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0177 iter count: 1.0 +[Apr 25, 2026 - 16:29:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0177 iter count: 1.0 +[Apr 25, 2026 - 16:29:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:29:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:29:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823388 38.867378 38.747208 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.994629 68.142578 67.945312 +util 96.250000 99.000000 93.000000 +[Apr 25, 2026 - 16:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 99 : avg iteration time 49.88 seconds +[Apr 25, 2026 - 16:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 99 : data loading time 0.00 +[Apr 25, 2026 - 16:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 99 : avg forward pass time 12.91 +[Apr 25, 2026 - 16:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 99 : backward pass time 11.55 +[Apr 25, 2026 - 16:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 99 : optimizer step time 0.97 +[Apr 25, 2026 - 16:29:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 99-------------------- +[Apr 25, 2026 - 16:29:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0175 iter count: 1.0 +[Apr 25, 2026 - 16:29:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0175 iter count: 1.0 +[Apr 25, 2026 - 16:29:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:29:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.823384 38.867386 38.747192 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.120117 133.134766 133.076172 +util 72.000000 77.000000 68.000000 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 100 : avg iteration time 178.65 seconds +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 100 : data loading time 0.00 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 100 : avg forward pass time 80.05 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 100 : backward pass time 9.13 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 100 : optimizer step time 0.34 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 100-------------------- +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 22666.1541 iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6126 iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 22666.1541iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.0492iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7768 iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.4085 iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 39689.7461iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.7280 iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 368401.9375iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6126iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.8944 iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 2413.6206iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.6062 iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 10020.4307iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.7550 iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 13495.0527iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.5867 iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 71205.9922iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0173 iter count: 1.0 +[Apr 25, 2026 - 16:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 25, 2026 - 16:33:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 18.92 seconds +[Apr 25, 2026 - 16:33:17 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 100 +[Apr 25, 2026 - 16:33:17 | INFO | fastgen.utils.checkpointer:save:301 ] Saving FSDP model to prefix /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000100 +[Apr 25, 2026 - 16:33:33 | SUCCESS | fastgen.utils.checkpointer:save:329 ] Model saved at iteration 100 +[Apr 25, 2026 - 16:33:33 | INFO | fastgen.trainer:validate:358 ] Validation iteration 100 +[Apr 25, 2026 - 16:33:34 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 16:33:36 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 2.2s +[Apr 25, 2026 - 16:33:39 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 16:33:41 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 16:33:44 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 16:33:46 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 16:33:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 16:33:52 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 16:33:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 16:33:57 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 16:34:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 16:34:02 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 16:34:05 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 16:34:07 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 16:34:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 16:34:12 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 16:34:16 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 16:34:17 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 16:34:21 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 16:34:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 16:34:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging val0 stats at iteration 100-------------------- +[Apr 25, 2026 - 16:34:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 25, 2026 - 16:34:43 | INFO | fastgen.callbacks.wandb:on_validation_end:683 ] Logged 10 val videos at iteration 100 +[Apr 25, 2026 - 16:35:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.089966 43.148026 43.002857 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.704590 70.896484 70.640625 +util 98.000000 100.000000 95.000000 +[Apr 25, 2026 - 16:35:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 101 : avg iteration time 151.30 seconds +[Apr 25, 2026 - 16:35:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 101 : data loading time 0.00 +[Apr 25, 2026 - 16:35:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 101 : avg forward pass time 11.08 +[Apr 25, 2026 - 16:35:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 101 : backward pass time 11.54 +[Apr 25, 2026 - 16:35:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 101 : optimizer step time 0.99 +[Apr 25, 2026 - 16:35:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 101-------------------- +[Apr 25, 2026 - 16:35:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0207 iter count: 1.0 +[Apr 25, 2026 - 16:35:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0207 iter count: 1.0 +[Apr 25, 2026 - 16:35:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:35:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:36:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.089714 43.147064 43.002834 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.763184 71.072266 70.660156 +util 97.500000 100.000000 95.000000 +[Apr 25, 2026 - 16:36:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 102 : avg iteration time 53.07 seconds +[Apr 25, 2026 - 16:36:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 102 : data loading time 0.00 +[Apr 25, 2026 - 16:36:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 102 : avg forward pass time 14.46 +[Apr 25, 2026 - 16:36:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 102 : backward pass time 11.58 +[Apr 25, 2026 - 16:36:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 102 : optimizer step time 0.99 +[Apr 25, 2026 - 16:36:22 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 102-------------------- +[Apr 25, 2026 - 16:36:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0215 iter count: 1.0 +[Apr 25, 2026 - 16:36:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0215 iter count: 1.0 +[Apr 25, 2026 - 16:36:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:36:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:37:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.089554 43.146416 43.002834 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.641113 70.994141 70.523438 +util 97.250000 99.000000 92.000000 +[Apr 25, 2026 - 16:37:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 103 : avg iteration time 47.06 seconds +[Apr 25, 2026 - 16:37:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 103 : data loading time 0.00 +[Apr 25, 2026 - 16:37:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 103 : avg forward pass time 11.49 +[Apr 25, 2026 - 16:37:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 103 : backward pass time 11.51 +[Apr 25, 2026 - 16:37:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 103 : optimizer step time 1.00 +[Apr 25, 2026 - 16:37:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 103-------------------- +[Apr 25, 2026 - 16:37:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0203 iter count: 1.0 +[Apr 25, 2026 - 16:37:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0203 iter count: 1.0 +[Apr 25, 2026 - 16:37:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:37:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:37:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.089563 43.146397 43.002857 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.729004 70.935547 70.660156 +util 97.750000 99.000000 96.000000 +[Apr 25, 2026 - 16:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 104 : avg iteration time 49.96 seconds +[Apr 25, 2026 - 16:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 104 : data loading time 0.00 +[Apr 25, 2026 - 16:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 104 : avg forward pass time 12.94 +[Apr 25, 2026 - 16:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 104 : backward pass time 11.55 +[Apr 25, 2026 - 16:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 104 : optimizer step time 0.99 +[Apr 25, 2026 - 16:37:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 104-------------------- +[Apr 25, 2026 - 16:37:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0205 iter count: 1.0 +[Apr 25, 2026 - 16:37:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0205 iter count: 1.0 +[Apr 25, 2026 - 16:37:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:37:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.089588 43.144478 43.003365 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.458984 135.458984 135.458984 +util 71.250000 76.000000 66.000000 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 105 : avg iteration time 172.51 seconds +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 105 : data loading time 0.00 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 105 : avg forward pass time 76.95 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 105 : backward pass time 9.17 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 105 : optimizer step time 0.35 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 105-------------------- +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 13887.8755 iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6591 iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 13887.8755iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.5631iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5503 iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.1066 iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 18749.9531iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.0060 iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 201433.9531iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6591iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.5820 iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 70548.5781iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.5791 iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 9491.7686iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.9137 iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 136962.8750iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.9492 iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 364.4543iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0198 iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:40:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 16:41:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.089702 43.144871 43.003433 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.533691 70.623047 70.503906 +util 97.250000 99.000000 95.000000 +[Apr 25, 2026 - 16:41:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 106 : avg iteration time 48.47 seconds +[Apr 25, 2026 - 16:41:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 106 : data loading time 0.00 +[Apr 25, 2026 - 16:41:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 106 : avg forward pass time 12.22 +[Apr 25, 2026 - 16:41:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 106 : backward pass time 11.52 +[Apr 25, 2026 - 16:41:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 106 : optimizer step time 0.98 +[Apr 25, 2026 - 16:41:40 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 106-------------------- +[Apr 25, 2026 - 16:41:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0184 iter count: 1.0 +[Apr 25, 2026 - 16:41:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0184 iter count: 1.0 +[Apr 25, 2026 - 16:41:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:41:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:42:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.089979 43.145844 43.003456 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.650879 70.679688 70.564453 +util 97.250000 99.000000 95.000000 +[Apr 25, 2026 - 16:42:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 107 : avg iteration time 52.80 seconds +[Apr 25, 2026 - 16:42:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 107 : data loading time 0.00 +[Apr 25, 2026 - 16:42:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 107 : avg forward pass time 14.39 +[Apr 25, 2026 - 16:42:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 107 : backward pass time 11.52 +[Apr 25, 2026 - 16:42:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 107 : optimizer step time 0.96 +[Apr 25, 2026 - 16:42:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 107-------------------- +[Apr 25, 2026 - 16:42:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0166 iter count: 1.0 +[Apr 25, 2026 - 16:42:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0166 iter count: 1.0 +[Apr 25, 2026 - 16:42:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:42:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:43:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.090471 43.147835 43.003433 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.645996 70.679688 70.544922 +util 97.500000 100.000000 94.000000 +[Apr 25, 2026 - 16:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 108 : avg iteration time 52.91 seconds +[Apr 25, 2026 - 16:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 108 : data loading time 0.00 +[Apr 25, 2026 - 16:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 108 : avg forward pass time 14.42 +[Apr 25, 2026 - 16:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 108 : backward pass time 11.53 +[Apr 25, 2026 - 16:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 108 : optimizer step time 0.97 +[Apr 25, 2026 - 16:43:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 108-------------------- +[Apr 25, 2026 - 16:43:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0157 iter count: 1.0 +[Apr 25, 2026 - 16:43:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0157 iter count: 1.0 +[Apr 25, 2026 - 16:43:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:43:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:44:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.090470 43.147820 43.003433 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.631348 70.660156 70.544922 +util 98.500000 100.000000 97.000000 +[Apr 25, 2026 - 16:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 109 : avg iteration time 49.92 seconds +[Apr 25, 2026 - 16:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 109 : data loading time 0.00 +[Apr 25, 2026 - 16:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 109 : avg forward pass time 12.92 +[Apr 25, 2026 - 16:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 109 : backward pass time 11.54 +[Apr 25, 2026 - 16:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 109 : optimizer step time 0.98 +[Apr 25, 2026 - 16:44:16 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 109-------------------- +[Apr 25, 2026 - 16:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0179 iter count: 1.0 +[Apr 25, 2026 - 16:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0179 iter count: 1.0 +[Apr 25, 2026 - 16:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:44:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.090361 43.147289 43.003441 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.425293 135.441406 135.419922 +util 66.250000 70.000000 61.000000 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 110 : avg iteration time 172.52 seconds +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 110 : data loading time 0.00 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 110 : avg forward pass time 76.97 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 110 : backward pass time 9.15 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 110 : optimizer step time 0.34 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 110-------------------- +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 25453.7915 iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6583 iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 25453.7915iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1372iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.3097 iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.7857 iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 59485.0781iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 13.7284iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 783363.4375iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6583iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.1456 iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 3988.8105iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.2561 iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 4975.3687iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.0583 iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 61.3521 iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.5982 iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 72870.8203iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0184 iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:47:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 16:47:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.090431 43.147537 43.003441 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.592285 70.601562 70.564453 +util 96.250000 99.000000 93.000000 +[Apr 25, 2026 - 16:47:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 111 : avg iteration time 51.23 seconds +[Apr 25, 2026 - 16:47:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 111 : data loading time 0.00 +[Apr 25, 2026 - 16:47:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 111 : avg forward pass time 13.61 +[Apr 25, 2026 - 16:47:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 111 : backward pass time 11.53 +[Apr 25, 2026 - 16:47:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 111 : optimizer step time 0.98 +[Apr 25, 2026 - 16:47:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 111-------------------- +[Apr 25, 2026 - 16:47:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0170 iter count: 1.0 +[Apr 25, 2026 - 16:47:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0170 iter count: 1.0 +[Apr 25, 2026 - 16:47:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:47:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:48:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.090730 43.148746 43.003441 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.582520 70.701172 70.542969 +util 97.750000 100.000000 95.000000 +[Apr 25, 2026 - 16:48:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 112 : avg iteration time 49.93 seconds +[Apr 25, 2026 - 16:48:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 112 : data loading time 0.00 +[Apr 25, 2026 - 16:48:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 112 : avg forward pass time 12.94 +[Apr 25, 2026 - 16:48:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 112 : backward pass time 11.53 +[Apr 25, 2026 - 16:48:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 112 : optimizer step time 0.98 +[Apr 25, 2026 - 16:48:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 112-------------------- +[Apr 25, 2026 - 16:48:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0165 iter count: 1.0 +[Apr 25, 2026 - 16:48:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0165 iter count: 1.0 +[Apr 25, 2026 - 16:48:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:48:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:49:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.090725 43.148746 43.003441 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.626465 70.701172 70.601562 +util 96.500000 100.000000 90.000000 +[Apr 25, 2026 - 16:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 113 : avg iteration time 47.03 seconds +[Apr 25, 2026 - 16:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 113 : data loading time 0.00 +[Apr 25, 2026 - 16:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 113 : avg forward pass time 11.48 +[Apr 25, 2026 - 16:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 113 : backward pass time 11.53 +[Apr 25, 2026 - 16:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 113 : optimizer step time 0.97 +[Apr 25, 2026 - 16:49:36 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 113-------------------- +[Apr 25, 2026 - 16:49:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0190 iter count: 1.0 +[Apr 25, 2026 - 16:49:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0190 iter count: 1.0 +[Apr 25, 2026 - 16:49:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:49:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:50:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.090731 43.148746 43.003441 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.582520 70.701172 70.542969 +util 95.750000 99.000000 91.000000 +[Apr 25, 2026 - 16:50:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 114 : avg iteration time 46.93 seconds +[Apr 25, 2026 - 16:50:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 114 : data loading time 0.00 +[Apr 25, 2026 - 16:50:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 114 : avg forward pass time 11.44 +[Apr 25, 2026 - 16:50:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 114 : backward pass time 11.54 +[Apr 25, 2026 - 16:50:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 114 : optimizer step time 0.97 +[Apr 25, 2026 - 16:50:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 114-------------------- +[Apr 25, 2026 - 16:50:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0178 iter count: 1.0 +[Apr 25, 2026 - 16:50:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0178 iter count: 1.0 +[Apr 25, 2026 - 16:50:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:50:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.090956 43.149635 43.003445 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.469238 135.500000 135.458984 +util 71.250000 78.000000 64.000000 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 115 : avg iteration time 175.82 seconds +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 115 : data loading time 0.00 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 115 : avg forward pass time 78.62 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 115 : backward pass time 9.15 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 115 : optimizer step time 0.37 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 115-------------------- +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 54503.3865 iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6408 iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 54503.3865iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.2286iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.9846 iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0689 iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 89776.2344iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 52.9437iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1380362.2500iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6408iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.9910 iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 159854.2188iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.7284 iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 698528.1875iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.9773 iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 2848.8098iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 1.9846 iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 52.9437 iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0173 iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:53:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 16:54:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.090960 43.149635 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.636230 70.740234 70.601562 +util 95.000000 99.000000 91.000000 +[Apr 25, 2026 - 16:54:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 116 : avg iteration time 54.26 seconds +[Apr 25, 2026 - 16:54:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 116 : data loading time 0.00 +[Apr 25, 2026 - 16:54:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 116 : avg forward pass time 15.10 +[Apr 25, 2026 - 16:54:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 116 : backward pass time 11.53 +[Apr 25, 2026 - 16:54:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 116 : optimizer step time 0.98 +[Apr 25, 2026 - 16:54:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 116-------------------- +[Apr 25, 2026 - 16:54:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0171 iter count: 1.0 +[Apr 25, 2026 - 16:54:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0171 iter count: 1.0 +[Apr 25, 2026 - 16:54:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:54:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:55:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.090964 43.149635 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.626465 70.640625 70.583984 +util 97.750000 99.000000 96.000000 +[Apr 25, 2026 - 16:55:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 117 : avg iteration time 47.06 seconds +[Apr 25, 2026 - 16:55:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 117 : data loading time 0.00 +[Apr 25, 2026 - 16:55:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 117 : avg forward pass time 11.50 +[Apr 25, 2026 - 16:55:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 117 : backward pass time 11.54 +[Apr 25, 2026 - 16:55:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 117 : optimizer step time 0.97 +[Apr 25, 2026 - 16:55:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 117-------------------- +[Apr 25, 2026 - 16:55:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0196 iter count: 1.0 +[Apr 25, 2026 - 16:55:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0196 iter count: 1.0 +[Apr 25, 2026 - 16:55:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:55:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 16:55:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.090963 43.149651 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.602051 70.720703 70.562500 +util 97.000000 100.000000 93.000000 +[Apr 25, 2026 - 16:55:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 118 : avg iteration time 47.09 seconds +[Apr 25, 2026 - 16:55:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 118 : data loading time 0.00 +[Apr 25, 2026 - 16:55:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 118 : avg forward pass time 11.49 +[Apr 25, 2026 - 16:55:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 118 : backward pass time 11.56 +[Apr 25, 2026 - 16:55:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 118 : optimizer step time 0.99 +[Apr 25, 2026 - 16:55:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 118-------------------- +[Apr 25, 2026 - 16:55:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0196 iter count: 1.0 +[Apr 25, 2026 - 16:55:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0196 iter count: 1.0 +[Apr 25, 2026 - 16:55:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:55:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:56:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.091088 43.150135 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.597168 70.701172 70.562500 +util 97.000000 99.000000 91.000000 +[Apr 25, 2026 - 16:56:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 119 : avg iteration time 49.94 seconds +[Apr 25, 2026 - 16:56:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 119 : data loading time 0.00 +[Apr 25, 2026 - 16:56:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 119 : avg forward pass time 12.95 +[Apr 25, 2026 - 16:56:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 119 : backward pass time 11.53 +[Apr 25, 2026 - 16:56:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 119 : optimizer step time 0.98 +[Apr 25, 2026 - 16:56:37 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 119-------------------- +[Apr 25, 2026 - 16:56:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0191 iter count: 1.0 +[Apr 25, 2026 - 16:56:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0191 iter count: 1.0 +[Apr 25, 2026 - 16:56:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:56:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.091366 43.151028 43.003460 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.454590 135.500000 135.439453 +util 71.250000 78.000000 64.000000 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 120 : avg iteration time 176.02 seconds +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 120 : data loading time 0.00 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 120 : avg forward pass time 78.75 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 120 : backward pass time 9.11 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 120 : optimizer step time 0.34 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 120-------------------- +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 47217.5967 iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6177 iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 47217.5967iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3674iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.0674 iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.8187 iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 67066.8906iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 62.4808iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 836915.8750iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6177iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.4048 iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 906.5971iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.1453 iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 73.0167 iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.1627 iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 30498.7617iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.1426 iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 29293.1934iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0180 iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 16:59:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 17:00:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.091425 43.151283 43.003483 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.533691 70.681641 70.484375 +util 98.250000 99.000000 96.000000 +[Apr 25, 2026 - 17:00:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 121 : avg iteration time 48.46 seconds +[Apr 25, 2026 - 17:00:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 121 : data loading time 0.00 +[Apr 25, 2026 - 17:00:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 121 : avg forward pass time 12.19 +[Apr 25, 2026 - 17:00:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 121 : backward pass time 11.56 +[Apr 25, 2026 - 17:00:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 121 : optimizer step time 0.98 +[Apr 25, 2026 - 17:00:22 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 121-------------------- +[Apr 25, 2026 - 17:00:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0225 iter count: 1.0 +[Apr 25, 2026 - 17:00:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0225 iter count: 1.0 +[Apr 25, 2026 - 17:00:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:00:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:01:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.091488 43.151512 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.543457 70.662109 70.503906 +util 94.500000 99.000000 90.000000 +[Apr 25, 2026 - 17:01:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 122 : avg iteration time 50.02 seconds +[Apr 25, 2026 - 17:01:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 122 : data loading time 0.00 +[Apr 25, 2026 - 17:01:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 122 : avg forward pass time 12.99 +[Apr 25, 2026 - 17:01:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 122 : backward pass time 11.52 +[Apr 25, 2026 - 17:01:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 122 : optimizer step time 0.98 +[Apr 25, 2026 - 17:01:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 122-------------------- +[Apr 25, 2026 - 17:01:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0164 iter count: 1.0 +[Apr 25, 2026 - 17:01:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0164 iter count: 1.0 +[Apr 25, 2026 - 17:01:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:01:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:02:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.091482 43.151512 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.558105 70.662109 70.523438 +util 98.250000 99.000000 96.000000 +[Apr 25, 2026 - 17:02:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 123 : avg iteration time 49.98 seconds +[Apr 25, 2026 - 17:02:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 123 : data loading time 0.00 +[Apr 25, 2026 - 17:02:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 123 : avg forward pass time 12.97 +[Apr 25, 2026 - 17:02:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 123 : backward pass time 11.54 +[Apr 25, 2026 - 17:02:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 123 : optimizer step time 0.98 +[Apr 25, 2026 - 17:02:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 123-------------------- +[Apr 25, 2026 - 17:02:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0198 iter count: 1.0 +[Apr 25, 2026 - 17:02:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0198 iter count: 1.0 +[Apr 25, 2026 - 17:02:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:02:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:02:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.091545 43.151764 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.719238 70.738281 70.662109 +util 98.000000 100.000000 94.000000 +[Apr 25, 2026 - 17:02:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 124 : avg iteration time 49.94 seconds +[Apr 25, 2026 - 17:02:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 124 : data loading time 0.00 +[Apr 25, 2026 - 17:02:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 124 : avg forward pass time 12.96 +[Apr 25, 2026 - 17:02:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 124 : backward pass time 11.52 +[Apr 25, 2026 - 17:02:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 124 : optimizer step time 0.98 +[Apr 25, 2026 - 17:02:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 124-------------------- +[Apr 25, 2026 - 17:02:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0195 iter count: 1.0 +[Apr 25, 2026 - 17:02:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0195 iter count: 1.0 +[Apr 25, 2026 - 17:02:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:02:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092032 43.153713 43.003460 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.483398 135.498047 135.478516 +util 70.500000 76.000000 66.000000 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 125 : avg iteration time 169.54 seconds +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 125 : data loading time 0.00 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 125 : avg forward pass time 75.48 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 125 : backward pass time 9.14 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 125 : optimizer step time 0.34 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 125-------------------- +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 2085821.3115 iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4836 iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 2085821.3115iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.2017iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7817 iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 9.5324 iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 6125497.5000iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.7752 iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 190444400.0000iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4836iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.7149 iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 12454.4639iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.7089 iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 1665.4625iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 7.4602 iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 3019140.2500iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.8864 iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 129679.5234iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0189 iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:05:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 17:06:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092037 43.153717 43.003498 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.348145 70.701172 70.230469 +util 96.500000 100.000000 92.000000 +[Apr 25, 2026 - 17:06:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 126 : avg iteration time 54.27 seconds +[Apr 25, 2026 - 17:06:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 126 : data loading time 0.00 +[Apr 25, 2026 - 17:06:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 126 : avg forward pass time 15.11 +[Apr 25, 2026 - 17:06:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 126 : backward pass time 11.52 +[Apr 25, 2026 - 17:06:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 126 : optimizer step time 0.98 +[Apr 25, 2026 - 17:06:36 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 126-------------------- +[Apr 25, 2026 - 17:06:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0165 iter count: 1.0 +[Apr 25, 2026 - 17:06:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0165 iter count: 1.0 +[Apr 25, 2026 - 17:06:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:06:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:07:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092031 43.153706 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.348145 70.759766 70.210938 +util 96.500000 99.000000 90.000000 +[Apr 25, 2026 - 17:07:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 127 : avg iteration time 46.95 seconds +[Apr 25, 2026 - 17:07:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 127 : data loading time 0.00 +[Apr 25, 2026 - 17:07:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 127 : avg forward pass time 11.46 +[Apr 25, 2026 - 17:07:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 127 : backward pass time 11.52 +[Apr 25, 2026 - 17:07:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 127 : optimizer step time 0.98 +[Apr 25, 2026 - 17:07:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 127-------------------- +[Apr 25, 2026 - 17:07:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0185 iter count: 1.0 +[Apr 25, 2026 - 17:07:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0185 iter count: 1.0 +[Apr 25, 2026 - 17:07:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:07:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:08:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.091987 43.153534 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.313965 70.623047 70.210938 +util 97.000000 100.000000 93.000000 +[Apr 25, 2026 - 17:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 128 : avg iteration time 47.07 seconds +[Apr 25, 2026 - 17:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 128 : data loading time 0.00 +[Apr 25, 2026 - 17:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 128 : avg forward pass time 11.51 +[Apr 25, 2026 - 17:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 128 : backward pass time 11.52 +[Apr 25, 2026 - 17:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 128 : optimizer step time 0.99 +[Apr 25, 2026 - 17:08:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 128-------------------- +[Apr 25, 2026 - 17:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0186 iter count: 1.0 +[Apr 25, 2026 - 17:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0186 iter count: 1.0 +[Apr 25, 2026 - 17:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:08:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:08:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092034 43.153698 43.003483 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.309082 70.603516 70.210938 +util 96.500000 100.000000 93.000000 +[Apr 25, 2026 - 17:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 129 : avg iteration time 47.07 seconds +[Apr 25, 2026 - 17:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 129 : data loading time 0.00 +[Apr 25, 2026 - 17:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 129 : avg forward pass time 11.49 +[Apr 25, 2026 - 17:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 129 : backward pass time 11.54 +[Apr 25, 2026 - 17:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 129 : optimizer step time 0.99 +[Apr 25, 2026 - 17:08:57 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 129-------------------- +[Apr 25, 2026 - 17:08:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0185 iter count: 1.0 +[Apr 25, 2026 - 17:08:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0185 iter count: 1.0 +[Apr 25, 2026 - 17:08:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:08:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:11:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092568 43.155869 43.003460 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.552246 135.576172 135.480469 +util 69.250000 74.000000 64.000000 +[Apr 25, 2026 - 17:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 130 : avg iteration time 172.72 seconds +[Apr 25, 2026 - 17:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 130 : data loading time 0.00 +[Apr 25, 2026 - 17:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 130 : avg forward pass time 77.11 +[Apr 25, 2026 - 17:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 130 : backward pass time 9.13 +[Apr 25, 2026 - 17:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 130 : optimizer step time 0.34 +[Apr 25, 2026 - 17:11:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 130-------------------- +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 43226.9202 iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6540 iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 43226.9202iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1197iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.3892 iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.7649 iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 63834.6758iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 16.0930iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 751424.5625iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6540iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.0705 iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 25364.2539iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.4622 iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 55515.1836iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 1.6461 iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 26.9011 iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.6771 iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 85330.2500iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0171 iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:11:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 17:12:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092592 43.155952 43.003483 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.519043 70.523438 70.505859 +util 98.250000 99.000000 97.000000 +[Apr 25, 2026 - 17:12:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 131 : avg iteration time 54.32 seconds +[Apr 25, 2026 - 17:12:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 131 : data loading time 0.00 +[Apr 25, 2026 - 17:12:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 131 : avg forward pass time 15.14 +[Apr 25, 2026 - 17:12:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 131 : backward pass time 11.52 +[Apr 25, 2026 - 17:12:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 131 : optimizer step time 0.99 +[Apr 25, 2026 - 17:12:44 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 131-------------------- +[Apr 25, 2026 - 17:12:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0200 iter count: 1.0 +[Apr 25, 2026 - 17:12:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0200 iter count: 1.0 +[Apr 25, 2026 - 17:12:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:12:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:13:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092644 43.156139 43.003502 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.558105 70.662109 70.523438 +util 96.250000 99.000000 93.000000 +[Apr 25, 2026 - 17:13:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 132 : avg iteration time 49.95 seconds +[Apr 25, 2026 - 17:13:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 132 : data loading time 0.00 +[Apr 25, 2026 - 17:13:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 132 : avg forward pass time 12.94 +[Apr 25, 2026 - 17:13:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 132 : backward pass time 11.55 +[Apr 25, 2026 - 17:13:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 132 : optimizer step time 0.98 +[Apr 25, 2026 - 17:13:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 132-------------------- +[Apr 25, 2026 - 17:13:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0177 iter count: 1.0 +[Apr 25, 2026 - 17:13:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0177 iter count: 1.0 +[Apr 25, 2026 - 17:13:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:13:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:14:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092636 43.156151 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.577637 70.623047 70.562500 +util 96.500000 100.000000 92.000000 +[Apr 25, 2026 - 17:14:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 133 : avg iteration time 49.92 seconds +[Apr 25, 2026 - 17:14:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 133 : data loading time 0.00 +[Apr 25, 2026 - 17:14:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 133 : avg forward pass time 12.94 +[Apr 25, 2026 - 17:14:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 133 : backward pass time 11.55 +[Apr 25, 2026 - 17:14:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 133 : optimizer step time 0.98 +[Apr 25, 2026 - 17:14:24 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 133-------------------- +[Apr 25, 2026 - 17:14:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0196 iter count: 1.0 +[Apr 25, 2026 - 17:14:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0196 iter count: 1.0 +[Apr 25, 2026 - 17:14:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:14:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:15:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092645 43.156139 43.003483 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.577637 70.623047 70.562500 +util 96.250000 99.000000 93.000000 +[Apr 25, 2026 - 17:15:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 134 : avg iteration time 52.88 seconds +[Apr 25, 2026 - 17:15:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 134 : data loading time 0.00 +[Apr 25, 2026 - 17:15:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 134 : avg forward pass time 14.41 +[Apr 25, 2026 - 17:15:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 134 : backward pass time 11.51 +[Apr 25, 2026 - 17:15:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 134 : optimizer step time 1.00 +[Apr 25, 2026 - 17:15:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 134-------------------- +[Apr 25, 2026 - 17:15:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0183 iter count: 1.0 +[Apr 25, 2026 - 17:15:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0183 iter count: 1.0 +[Apr 25, 2026 - 17:15:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:15:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092443 43.155376 43.003460 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.474121 135.478516 135.460938 +util 68.000000 73.000000 63.000000 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 135 : avg iteration time 169.80 seconds +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 135 : data loading time 0.00 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 135 : avg forward pass time 75.66 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 135 : backward pass time 9.12 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 135 : optimizer step time 0.34 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 135-------------------- +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 38858.9368 iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6506 iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 38858.9368iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.7826iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.8751 iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.7582 iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 67450.7734iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 5.7559 iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 741474.1250iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6506iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.3215 iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 41898.7422iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 1.0306 iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 7.8557 iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.7483 iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 13315.5635iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.3375 iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 107.2349iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0166 iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:18:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 17:18:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092458 43.155415 43.003483 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.357910 70.681641 70.250000 +util 97.250000 99.000000 92.000000 +[Apr 25, 2026 - 17:18:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 136 : avg iteration time 51.29 seconds +[Apr 25, 2026 - 17:18:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 136 : data loading time 0.00 +[Apr 25, 2026 - 17:18:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 136 : avg forward pass time 13.63 +[Apr 25, 2026 - 17:18:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 136 : backward pass time 11.51 +[Apr 25, 2026 - 17:18:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 136 : optimizer step time 0.99 +[Apr 25, 2026 - 17:18:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 136-------------------- +[Apr 25, 2026 - 17:18:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0189 iter count: 1.0 +[Apr 25, 2026 - 17:18:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0189 iter count: 1.0 +[Apr 25, 2026 - 17:18:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:18:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:19:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092781 43.156731 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.509277 70.642578 70.464844 +util 96.500000 100.000000 92.000000 +[Apr 25, 2026 - 17:19:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 137 : avg iteration time 50.03 seconds +[Apr 25, 2026 - 17:19:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 137 : data loading time 0.00 +[Apr 25, 2026 - 17:19:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 137 : avg forward pass time 12.98 +[Apr 25, 2026 - 17:19:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 137 : backward pass time 11.55 +[Apr 25, 2026 - 17:19:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 137 : optimizer step time 0.98 +[Apr 25, 2026 - 17:19:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 137-------------------- +[Apr 25, 2026 - 17:19:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0201 iter count: 1.0 +[Apr 25, 2026 - 17:19:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0201 iter count: 1.0 +[Apr 25, 2026 - 17:19:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:19:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:20:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092787 43.156731 43.003483 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.519043 70.740234 70.445312 +util 97.500000 99.000000 96.000000 +[Apr 25, 2026 - 17:20:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 138 : avg iteration time 49.92 seconds +[Apr 25, 2026 - 17:20:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 138 : data loading time 0.00 +[Apr 25, 2026 - 17:20:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 138 : avg forward pass time 12.95 +[Apr 25, 2026 - 17:20:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 138 : backward pass time 11.53 +[Apr 25, 2026 - 17:20:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 138 : optimizer step time 0.98 +[Apr 25, 2026 - 17:20:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 138-------------------- +[Apr 25, 2026 - 17:20:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0187 iter count: 1.0 +[Apr 25, 2026 - 17:20:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0187 iter count: 1.0 +[Apr 25, 2026 - 17:20:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:20:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:21:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092781 43.156731 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.597168 70.759766 70.542969 +util 96.500000 100.000000 93.000000 +[Apr 25, 2026 - 17:21:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 139 : avg iteration time 49.98 seconds +[Apr 25, 2026 - 17:21:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 139 : data loading time 0.00 +[Apr 25, 2026 - 17:21:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 139 : avg forward pass time 12.96 +[Apr 25, 2026 - 17:21:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 139 : backward pass time 11.54 +[Apr 25, 2026 - 17:21:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 139 : optimizer step time 0.97 +[Apr 25, 2026 - 17:21:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 139-------------------- +[Apr 25, 2026 - 17:21:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0183 iter count: 1.0 +[Apr 25, 2026 - 17:21:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0183 iter count: 1.0 +[Apr 25, 2026 - 17:21:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:21:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092923 43.157299 43.003460 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.443359 135.460938 135.437500 +util 68.250000 77.000000 60.000000 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 140 : avg iteration time 175.95 seconds +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 140 : data loading time 0.00 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 140 : avg forward pass time 78.70 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 140 : backward pass time 9.16 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 140 : optimizer step time 0.34 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 140-------------------- +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 56264.1235 iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6027 iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 56264.1235iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3540iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.3535 iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.8458 iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 89848.8594iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 14.9855iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 883528.6250iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6027iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.5054 iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 60525.5820iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.1315 iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 28650.1719iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.4906 iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 58761.0000iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.4396 iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 971.8856iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0191 iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:24:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 17:25:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092923 43.157299 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.377441 70.642578 70.289062 +util 98.000000 99.000000 97.000000 +[Apr 25, 2026 - 17:25:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 141 : avg iteration time 51.29 seconds +[Apr 25, 2026 - 17:25:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 141 : data loading time 0.00 +[Apr 25, 2026 - 17:25:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 141 : avg forward pass time 13.61 +[Apr 25, 2026 - 17:25:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 141 : backward pass time 11.55 +[Apr 25, 2026 - 17:25:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 141 : optimizer step time 0.97 +[Apr 25, 2026 - 17:25:15 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 141-------------------- +[Apr 25, 2026 - 17:25:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0193 iter count: 1.0 +[Apr 25, 2026 - 17:25:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0193 iter count: 1.0 +[Apr 25, 2026 - 17:25:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:25:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:26:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093076 43.157879 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.353027 70.486328 70.308594 +util 97.000000 100.000000 93.000000 +[Apr 25, 2026 - 17:26:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 142 : avg iteration time 49.88 seconds +[Apr 25, 2026 - 17:26:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 142 : data loading time 0.00 +[Apr 25, 2026 - 17:26:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 142 : avg forward pass time 12.92 +[Apr 25, 2026 - 17:26:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 142 : backward pass time 11.54 +[Apr 25, 2026 - 17:26:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 142 : optimizer step time 0.97 +[Apr 25, 2026 - 17:26:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 142-------------------- +[Apr 25, 2026 - 17:26:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0186 iter count: 1.0 +[Apr 25, 2026 - 17:26:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0186 iter count: 1.0 +[Apr 25, 2026 - 17:26:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:26:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:26:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093084 43.157925 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.348145 70.466797 70.308594 +util 97.500000 100.000000 95.000000 +[Apr 25, 2026 - 17:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 143 : avg iteration time 49.90 seconds +[Apr 25, 2026 - 17:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 143 : data loading time 0.00 +[Apr 25, 2026 - 17:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 143 : avg forward pass time 12.92 +[Apr 25, 2026 - 17:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 143 : backward pass time 11.53 +[Apr 25, 2026 - 17:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 143 : optimizer step time 0.97 +[Apr 25, 2026 - 17:26:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 143-------------------- +[Apr 25, 2026 - 17:26:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0172 iter count: 1.0 +[Apr 25, 2026 - 17:26:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0172 iter count: 1.0 +[Apr 25, 2026 - 17:26:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:26:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:27:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093090 43.157925 43.003464 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.348145 70.466797 70.308594 +util 99.000000 100.000000 98.000000 +[Apr 25, 2026 - 17:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 144 : avg iteration time 47.03 seconds +[Apr 25, 2026 - 17:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 144 : data loading time 0.00 +[Apr 25, 2026 - 17:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 144 : avg forward pass time 11.48 +[Apr 25, 2026 - 17:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 144 : backward pass time 11.53 +[Apr 25, 2026 - 17:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 144 : optimizer step time 0.98 +[Apr 25, 2026 - 17:27:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 144-------------------- +[Apr 25, 2026 - 17:27:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0193 iter count: 1.0 +[Apr 25, 2026 - 17:27:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0193 iter count: 1.0 +[Apr 25, 2026 - 17:27:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:27:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093069 43.157883 43.003460 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.488770 135.578125 135.458984 +util 66.000000 70.000000 61.000000 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 145 : avg iteration time 172.52 seconds +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 145 : data loading time 0.00 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 145 : avg forward pass time 76.98 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 145 : backward pass time 9.15 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 145 : optimizer step time 0.34 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 145-------------------- +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 27198.9385 iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6342 iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 27198.9385iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.6769iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 3.0844 iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0377 iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 60431.8516iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 477.5909iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1296722.8750iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6342iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.7693 iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 13885.6133iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.8523 iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2218.3340iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.8521 iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 16387.6426iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.6154 iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1381.3992iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0178 iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:30:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 17:31:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093086 43.157928 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.387207 70.623047 70.308594 +util 96.750000 100.000000 92.000000 +[Apr 25, 2026 - 17:31:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 146 : avg iteration time 51.35 seconds +[Apr 25, 2026 - 17:31:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 146 : data loading time 0.00 +[Apr 25, 2026 - 17:31:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 146 : avg forward pass time 13.63 +[Apr 25, 2026 - 17:31:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 146 : backward pass time 11.55 +[Apr 25, 2026 - 17:31:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 146 : optimizer step time 0.98 +[Apr 25, 2026 - 17:31:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 146-------------------- +[Apr 25, 2026 - 17:31:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0211 iter count: 1.0 +[Apr 25, 2026 - 17:31:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0211 iter count: 1.0 +[Apr 25, 2026 - 17:31:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:31:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:32:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093171 43.158268 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.309082 70.603516 70.210938 +util 96.750000 100.000000 92.000000 +[Apr 25, 2026 - 17:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 147 : avg iteration time 52.80 seconds +[Apr 25, 2026 - 17:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 147 : data loading time 0.00 +[Apr 25, 2026 - 17:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 147 : avg forward pass time 14.37 +[Apr 25, 2026 - 17:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 147 : backward pass time 11.55 +[Apr 25, 2026 - 17:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 147 : optimizer step time 0.98 +[Apr 25, 2026 - 17:32:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 147-------------------- +[Apr 25, 2026 - 17:32:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0197 iter count: 1.0 +[Apr 25, 2026 - 17:32:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0197 iter count: 1.0 +[Apr 25, 2026 - 17:32:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:32:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:33:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093183 43.158314 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.299316 70.505859 70.230469 +util 96.000000 99.000000 90.000000 +[Apr 25, 2026 - 17:33:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 148 : avg iteration time 49.84 seconds +[Apr 25, 2026 - 17:33:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 148 : data loading time 0.00 +[Apr 25, 2026 - 17:33:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 148 : avg forward pass time 12.91 +[Apr 25, 2026 - 17:33:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 148 : backward pass time 11.52 +[Apr 25, 2026 - 17:33:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 148 : optimizer step time 0.98 +[Apr 25, 2026 - 17:33:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 148-------------------- +[Apr 25, 2026 - 17:33:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0163 iter count: 1.0 +[Apr 25, 2026 - 17:33:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0163 iter count: 1.0 +[Apr 25, 2026 - 17:33:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:33:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:33:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093167 43.158268 43.003468 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.279785 70.486328 70.210938 +util 97.000000 100.000000 93.000000 +[Apr 25, 2026 - 17:33:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 149 : avg iteration time 46.97 seconds +[Apr 25, 2026 - 17:33:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 149 : data loading time 0.00 +[Apr 25, 2026 - 17:33:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 149 : avg forward pass time 11.47 +[Apr 25, 2026 - 17:33:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 149 : backward pass time 11.51 +[Apr 25, 2026 - 17:33:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 149 : optimizer step time 1.00 +[Apr 25, 2026 - 17:33:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 149-------------------- +[Apr 25, 2026 - 17:33:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0194 iter count: 1.0 +[Apr 25, 2026 - 17:33:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0194 iter count: 1.0 +[Apr 25, 2026 - 17:33:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:33:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093054 43.157810 43.003471 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.483887 135.498047 135.441406 +util 66.250000 72.000000 62.000000 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 150 : avg iteration time 172.66 seconds +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 150 : data loading time 0.00 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 150 : avg forward pass time 77.07 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 150 : backward pass time 9.14 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 150 : optimizer step time 0.34 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 150-------------------- +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 55759.2788 iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6474 iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 55759.2788iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5124iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.4340 iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.8106 iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 87156.3672iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 17.6012iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 823325.6250iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6474iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.5138 iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1127.3210iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.7162 iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 12487.1934iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.8321 iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 15743.4062iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.1154 iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 27745.3496iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0181 iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:36:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 17:37:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093109 43.158043 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.636230 70.740234 70.601562 +util 98.250000 100.000000 96.000000 +[Apr 25, 2026 - 17:37:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 151 : avg iteration time 51.45 seconds +[Apr 25, 2026 - 17:37:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 151 : data loading time 0.00 +[Apr 25, 2026 - 17:37:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 151 : avg forward pass time 13.67 +[Apr 25, 2026 - 17:37:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 151 : backward pass time 11.59 +[Apr 25, 2026 - 17:37:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 151 : optimizer step time 0.98 +[Apr 25, 2026 - 17:37:39 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 151-------------------- +[Apr 25, 2026 - 17:37:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0203 iter count: 1.0 +[Apr 25, 2026 - 17:37:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0203 iter count: 1.0 +[Apr 25, 2026 - 17:37:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:37:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:38:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093109 43.158043 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.558105 70.720703 70.503906 +util 96.750000 100.000000 92.000000 +[Apr 25, 2026 - 17:38:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 152 : avg iteration time 52.88 seconds +[Apr 25, 2026 - 17:38:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 152 : data loading time 0.00 +[Apr 25, 2026 - 17:38:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 152 : avg forward pass time 14.40 +[Apr 25, 2026 - 17:38:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 152 : backward pass time 11.54 +[Apr 25, 2026 - 17:38:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 152 : optimizer step time 0.98 +[Apr 25, 2026 - 17:38:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 152-------------------- +[Apr 25, 2026 - 17:38:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0201 iter count: 1.0 +[Apr 25, 2026 - 17:38:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0201 iter count: 1.0 +[Apr 25, 2026 - 17:38:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:38:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:39:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093114 43.158043 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.543457 70.662109 70.503906 +util 97.250000 99.000000 95.000000 +[Apr 25, 2026 - 17:39:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 153 : avg iteration time 49.91 seconds +[Apr 25, 2026 - 17:39:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 153 : data loading time 0.00 +[Apr 25, 2026 - 17:39:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 153 : avg forward pass time 12.91 +[Apr 25, 2026 - 17:39:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 153 : backward pass time 11.56 +[Apr 25, 2026 - 17:39:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 153 : optimizer step time 0.99 +[Apr 25, 2026 - 17:39:22 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 153-------------------- +[Apr 25, 2026 - 17:39:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0178 iter count: 1.0 +[Apr 25, 2026 - 17:39:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0178 iter count: 1.0 +[Apr 25, 2026 - 17:39:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:39:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:40:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093111 43.158051 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.543457 70.662109 70.503906 +util 98.250000 99.000000 97.000000 +[Apr 25, 2026 - 17:40:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 154 : avg iteration time 47.02 seconds +[Apr 25, 2026 - 17:40:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 154 : data loading time 0.00 +[Apr 25, 2026 - 17:40:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 154 : avg forward pass time 11.48 +[Apr 25, 2026 - 17:40:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 154 : backward pass time 11.54 +[Apr 25, 2026 - 17:40:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 154 : optimizer step time 0.99 +[Apr 25, 2026 - 17:40:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 154-------------------- +[Apr 25, 2026 - 17:40:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0206 iter count: 1.0 +[Apr 25, 2026 - 17:40:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0206 iter count: 1.0 +[Apr 25, 2026 - 17:40:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:40:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093202 43.158386 43.003464 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.453125 135.457031 135.441406 +util 68.250000 76.000000 61.000000 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 155 : avg iteration time 169.72 seconds +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 155 : data loading time 0.00 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 155 : avg forward pass time 75.58 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 155 : backward pass time 9.14 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 155 : optimizer step time 0.34 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 155-------------------- +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 132836.1851 iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5569 iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 132836.1851iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5573iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.8679 iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.6320 iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 253677.5000iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 41.9212iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 4256923.0000iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5569iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.1213 iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 3799.7593iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.6742 iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 11480.5283iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.4042 iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 905.4305iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.3260 iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 42276.1953iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0196 iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:42:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 17:43:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093252 43.158600 43.003464 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.675293 70.699219 70.603516 +util 97.000000 99.000000 91.000000 +[Apr 25, 2026 - 17:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 156 : avg iteration time 54.31 seconds +[Apr 25, 2026 - 17:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 156 : data loading time 0.00 +[Apr 25, 2026 - 17:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 156 : avg forward pass time 15.10 +[Apr 25, 2026 - 17:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 156 : backward pass time 11.53 +[Apr 25, 2026 - 17:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 156 : optimizer step time 1.00 +[Apr 25, 2026 - 17:43:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 156-------------------- +[Apr 25, 2026 - 17:43:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0199 iter count: 1.0 +[Apr 25, 2026 - 17:43:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0199 iter count: 1.0 +[Apr 25, 2026 - 17:43:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:43:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:44:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093256 43.158604 43.003464 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.694824 70.699219 70.681641 +util 96.250000 99.000000 93.000000 +[Apr 25, 2026 - 17:44:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 157 : avg iteration time 49.94 seconds +[Apr 25, 2026 - 17:44:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 157 : data loading time 0.00 +[Apr 25, 2026 - 17:44:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 157 : avg forward pass time 12.95 +[Apr 25, 2026 - 17:44:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 157 : backward pass time 11.52 +[Apr 25, 2026 - 17:44:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 157 : optimizer step time 0.99 +[Apr 25, 2026 - 17:44:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 157-------------------- +[Apr 25, 2026 - 17:44:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0190 iter count: 1.0 +[Apr 25, 2026 - 17:44:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0190 iter count: 1.0 +[Apr 25, 2026 - 17:44:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:44:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:45:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093256 43.158604 43.003487 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.748535 70.816406 70.544922 +util 98.000000 100.000000 94.000000 +[Apr 25, 2026 - 17:45:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 158 : avg iteration time 49.89 seconds +[Apr 25, 2026 - 17:45:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 158 : data loading time 0.00 +[Apr 25, 2026 - 17:45:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 158 : avg forward pass time 12.93 +[Apr 25, 2026 - 17:45:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 158 : backward pass time 11.53 +[Apr 25, 2026 - 17:45:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 158 : optimizer step time 0.98 +[Apr 25, 2026 - 17:45:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 158-------------------- +[Apr 25, 2026 - 17:45:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0188 iter count: 1.0 +[Apr 25, 2026 - 17:45:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0188 iter count: 1.0 +[Apr 25, 2026 - 17:45:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:45:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:46:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093264 43.158604 43.003471 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.792480 70.855469 70.603516 +util 97.500000 100.000000 95.000000 +[Apr 25, 2026 - 17:46:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 159 : avg iteration time 49.93 seconds +[Apr 25, 2026 - 17:46:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 159 : data loading time 0.00 +[Apr 25, 2026 - 17:46:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 159 : avg forward pass time 12.93 +[Apr 25, 2026 - 17:46:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 159 : backward pass time 11.53 +[Apr 25, 2026 - 17:46:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 159 : optimizer step time 1.00 +[Apr 25, 2026 - 17:46:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 159-------------------- +[Apr 25, 2026 - 17:46:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0204 iter count: 1.0 +[Apr 25, 2026 - 17:46:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0204 iter count: 1.0 +[Apr 25, 2026 - 17:46:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:46:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092875 43.157089 43.003464 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.488770 135.498047 135.460938 +util 68.500000 74.000000 62.000000 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 160 : avg iteration time 175.69 seconds +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 160 : data loading time 0.00 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 160 : avg forward pass time 78.60 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 160 : backward pass time 9.12 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 160 : optimizer step time 0.34 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 160-------------------- +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 52192.8181 iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6617 iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 52192.8181iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1095iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.1959 iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.7494 iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 65525.0391iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 10.9322iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 728490.3750iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6617iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.9977 iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 162002.5000iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.1533 iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 29926.8379iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.6283 iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1417.4039iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.8443 iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 2183.3503iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0175 iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:49:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 17:50:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092883 43.157089 43.003464 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.392090 70.583984 70.328125 +util 96.500000 100.000000 93.000000 +[Apr 25, 2026 - 17:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 161 : avg iteration time 48.48 seconds +[Apr 25, 2026 - 17:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 161 : data loading time 0.00 +[Apr 25, 2026 - 17:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 161 : avg forward pass time 12.23 +[Apr 25, 2026 - 17:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 161 : backward pass time 11.52 +[Apr 25, 2026 - 17:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 161 : optimizer step time 0.97 +[Apr 25, 2026 - 17:50:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 161-------------------- +[Apr 25, 2026 - 17:50:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0194 iter count: 1.0 +[Apr 25, 2026 - 17:50:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0194 iter count: 1.0 +[Apr 25, 2026 - 17:50:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:50:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:50:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093167 43.158272 43.003464 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.230957 70.583984 70.113281 +util 96.750000 100.000000 92.000000 +[Apr 25, 2026 - 17:50:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 162 : avg iteration time 49.92 seconds +[Apr 25, 2026 - 17:50:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 162 : data loading time 0.00 +[Apr 25, 2026 - 17:50:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 162 : avg forward pass time 12.93 +[Apr 25, 2026 - 17:50:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 162 : backward pass time 11.53 +[Apr 25, 2026 - 17:50:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 162 : optimizer step time 0.98 +[Apr 25, 2026 - 17:50:57 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 162-------------------- +[Apr 25, 2026 - 17:50:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0184 iter count: 1.0 +[Apr 25, 2026 - 17:50:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0184 iter count: 1.0 +[Apr 25, 2026 - 17:50:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:50:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:51:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093360 43.159019 43.003464 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.538574 70.583984 70.523438 +util 95.500000 99.000000 92.000000 +[Apr 25, 2026 - 17:51:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 163 : avg iteration time 49.87 seconds +[Apr 25, 2026 - 17:51:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 163 : data loading time 0.00 +[Apr 25, 2026 - 17:51:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 163 : avg forward pass time 12.92 +[Apr 25, 2026 - 17:51:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 163 : backward pass time 11.51 +[Apr 25, 2026 - 17:51:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 163 : optimizer step time 0.98 +[Apr 25, 2026 - 17:51:47 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 163-------------------- +[Apr 25, 2026 - 17:51:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0187 iter count: 1.0 +[Apr 25, 2026 - 17:51:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0187 iter count: 1.0 +[Apr 25, 2026 - 17:51:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:51:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:52:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093354 43.159019 43.003464 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.323730 70.662109 70.210938 +util 97.750000 99.000000 94.000000 +[Apr 25, 2026 - 17:52:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 164 : avg iteration time 49.91 seconds +[Apr 25, 2026 - 17:52:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 164 : data loading time 0.00 +[Apr 25, 2026 - 17:52:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 164 : avg forward pass time 12.94 +[Apr 25, 2026 - 17:52:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 164 : backward pass time 11.53 +[Apr 25, 2026 - 17:52:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 164 : optimizer step time 0.98 +[Apr 25, 2026 - 17:52:37 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 164-------------------- +[Apr 25, 2026 - 17:52:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0197 iter count: 1.0 +[Apr 25, 2026 - 17:52:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0197 iter count: 1.0 +[Apr 25, 2026 - 17:52:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:52:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092826 43.156879 43.003468 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.488770 135.498047 135.460938 +util 68.750000 75.000000 61.000000 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 165 : avg iteration time 172.97 seconds +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 165 : data loading time 0.00 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 165 : avg forward pass time 77.22 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 165 : backward pass time 9.14 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 165 : optimizer step time 0.34 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 165-------------------- +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 284372.4629 iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6434 iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 284372.4629iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 5.2406iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 3.2191 iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.9285 iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 360700.9688iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 625.2869iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 7701987.0000iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6434iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.2255 iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 34576.9648iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.1092 iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 202466.2500iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.3873 iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 875.2816iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.6565 iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 11080.5264iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0215 iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:55:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 17:56:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092849 43.156994 43.003468 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.343262 70.623047 70.250000 +util 97.250000 100.000000 94.000000 +[Apr 25, 2026 - 17:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 166 : avg iteration time 54.35 seconds +[Apr 25, 2026 - 17:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 166 : data loading time 0.00 +[Apr 25, 2026 - 17:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 166 : avg forward pass time 15.12 +[Apr 25, 2026 - 17:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 166 : backward pass time 11.56 +[Apr 25, 2026 - 17:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 166 : optimizer step time 0.99 +[Apr 25, 2026 - 17:56:24 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 166-------------------- +[Apr 25, 2026 - 17:56:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0229 iter count: 1.0 +[Apr 25, 2026 - 17:56:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0229 iter count: 1.0 +[Apr 25, 2026 - 17:56:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:56:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:57:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093154 43.158184 43.003487 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.562988 70.623047 70.542969 +util 97.000000 99.000000 91.000000 +[Apr 25, 2026 - 17:57:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 167 : avg iteration time 52.85 seconds +[Apr 25, 2026 - 17:57:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 167 : data loading time 0.00 +[Apr 25, 2026 - 17:57:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 167 : avg forward pass time 14.38 +[Apr 25, 2026 - 17:57:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 167 : backward pass time 11.55 +[Apr 25, 2026 - 17:57:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 167 : optimizer step time 0.99 +[Apr 25, 2026 - 17:57:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 167-------------------- +[Apr 25, 2026 - 17:57:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0219 iter count: 1.0 +[Apr 25, 2026 - 17:57:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0219 iter count: 1.0 +[Apr 25, 2026 - 17:57:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:57:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:58:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093148 43.158184 43.003468 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.709473 70.738281 70.623047 +util 97.750000 99.000000 96.000000 +[Apr 25, 2026 - 17:58:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 168 : avg iteration time 49.96 seconds +[Apr 25, 2026 - 17:58:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 168 : data loading time 0.00 +[Apr 25, 2026 - 17:58:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 168 : avg forward pass time 12.95 +[Apr 25, 2026 - 17:58:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 168 : backward pass time 11.52 +[Apr 25, 2026 - 17:58:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 168 : optimizer step time 0.99 +[Apr 25, 2026 - 17:58:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 168-------------------- +[Apr 25, 2026 - 17:58:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0222 iter count: 1.0 +[Apr 25, 2026 - 17:58:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0222 iter count: 1.0 +[Apr 25, 2026 - 17:58:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:58:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 17:58:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093217 43.158424 43.003490 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.689941 70.718750 70.603516 +util 96.750000 100.000000 92.000000 +[Apr 25, 2026 - 17:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 169 : avg iteration time 49.88 seconds +[Apr 25, 2026 - 17:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 169 : data loading time 0.00 +[Apr 25, 2026 - 17:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 169 : avg forward pass time 12.93 +[Apr 25, 2026 - 17:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 169 : backward pass time 11.52 +[Apr 25, 2026 - 17:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 169 : optimizer step time 0.99 +[Apr 25, 2026 - 17:58:57 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 169-------------------- +[Apr 25, 2026 - 17:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0190 iter count: 1.0 +[Apr 25, 2026 - 17:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0190 iter count: 1.0 +[Apr 25, 2026 - 17:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 17:58:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092896 43.156757 43.003895 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.493652 135.498047 135.480469 +util 72.500000 79.000000 65.000000 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 170 : avg iteration time 172.19 seconds +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 170 : data loading time 0.00 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 170 : avg forward pass time 76.85 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 170 : backward pass time 9.11 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 170 : optimizer step time 0.34 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 170-------------------- +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 845373.9399 iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6194 iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 845373.9399iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.7165iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.7137 iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.5276 iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 1077116.3750iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 30.7935iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 25525760.0000iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6194iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 8.5276 iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 25525760.0000iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 7.0915 iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 1443986.8750iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.0044 iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 22222.3945iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.3264 iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 104.8780iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0238 iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:01:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 18:02:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092964 43.157009 43.003880 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.270020 70.623047 70.152344 +util 97.000000 100.000000 94.000000 +[Apr 25, 2026 - 18:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 171 : avg iteration time 51.30 seconds +[Apr 25, 2026 - 18:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 171 : data loading time 0.00 +[Apr 25, 2026 - 18:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 171 : avg forward pass time 13.63 +[Apr 25, 2026 - 18:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 171 : backward pass time 11.52 +[Apr 25, 2026 - 18:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 171 : optimizer step time 1.00 +[Apr 25, 2026 - 18:02:40 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 171-------------------- +[Apr 25, 2026 - 18:02:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0213 iter count: 1.0 +[Apr 25, 2026 - 18:02:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0213 iter count: 1.0 +[Apr 25, 2026 - 18:02:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:02:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:03:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093202 43.157978 43.003872 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.323730 70.544922 70.250000 +util 97.500000 99.000000 93.000000 +[Apr 25, 2026 - 18:03:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 172 : avg iteration time 49.88 seconds +[Apr 25, 2026 - 18:03:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 172 : data loading time 0.00 +[Apr 25, 2026 - 18:03:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 172 : avg forward pass time 12.90 +[Apr 25, 2026 - 18:03:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 172 : backward pass time 11.57 +[Apr 25, 2026 - 18:03:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 172 : optimizer step time 0.98 +[Apr 25, 2026 - 18:03:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 172-------------------- +[Apr 25, 2026 - 18:03:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0276 iter count: 1.0 +[Apr 25, 2026 - 18:03:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0276 iter count: 1.0 +[Apr 25, 2026 - 18:03:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:03:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:04:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093196 43.157978 43.003872 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.313965 70.564453 70.230469 +util 95.250000 99.000000 91.000000 +[Apr 25, 2026 - 18:04:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 173 : avg iteration time 49.90 seconds +[Apr 25, 2026 - 18:04:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 173 : data loading time 0.00 +[Apr 25, 2026 - 18:04:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 173 : avg forward pass time 12.91 +[Apr 25, 2026 - 18:04:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 173 : backward pass time 11.55 +[Apr 25, 2026 - 18:04:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 173 : optimizer step time 0.99 +[Apr 25, 2026 - 18:04:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 173-------------------- +[Apr 25, 2026 - 18:04:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0257 iter count: 1.0 +[Apr 25, 2026 - 18:04:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0257 iter count: 1.0 +[Apr 25, 2026 - 18:04:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:04:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:05:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093260 43.158207 43.003872 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.323730 70.603516 70.230469 +util 98.000000 99.000000 96.000000 +[Apr 25, 2026 - 18:05:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 174 : avg iteration time 49.86 seconds +[Apr 25, 2026 - 18:05:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 174 : data loading time 0.00 +[Apr 25, 2026 - 18:05:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 174 : avg forward pass time 12.91 +[Apr 25, 2026 - 18:05:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 174 : backward pass time 11.53 +[Apr 25, 2026 - 18:05:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 174 : optimizer step time 0.98 +[Apr 25, 2026 - 18:05:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 174-------------------- +[Apr 25, 2026 - 18:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0240 iter count: 1.0 +[Apr 25, 2026 - 18:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0240 iter count: 1.0 +[Apr 25, 2026 - 18:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:05:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093035 43.157177 43.003967 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.439941 135.500000 135.419922 +util 70.250000 76.000000 66.000000 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 175 : avg iteration time 169.79 seconds +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 175 : data loading time 0.00 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 175 : avg forward pass time 75.63 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 175 : backward pass time 9.12 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 175 : optimizer step time 0.34 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 175-------------------- +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 121349.4531 iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5460 iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 121349.4531iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.8180iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5399 iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.3721 iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 286426.6875iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.9439 iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2531169.7500iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5460iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.8698 iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 125448.1562iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.5939 iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 1323.0621iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 6.3350 iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 318069.6875iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.9403 iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 19545.8438iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0250 iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:08:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 18:08:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093143 43.157612 43.003967 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.284668 70.681641 70.152344 +util 96.750000 99.000000 94.000000 +[Apr 25, 2026 - 18:08:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 176 : avg iteration time 51.37 seconds +[Apr 25, 2026 - 18:08:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 176 : data loading time 0.00 +[Apr 25, 2026 - 18:08:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 176 : avg forward pass time 13.67 +[Apr 25, 2026 - 18:08:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 176 : backward pass time 11.52 +[Apr 25, 2026 - 18:08:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 176 : optimizer step time 0.99 +[Apr 25, 2026 - 18:08:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 176-------------------- +[Apr 25, 2026 - 18:08:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0250 iter count: 1.0 +[Apr 25, 2026 - 18:08:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0250 iter count: 1.0 +[Apr 25, 2026 - 18:08:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:08:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:09:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093380 43.158581 43.003967 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.348145 70.642578 70.250000 +util 98.500000 100.000000 97.000000 +[Apr 25, 2026 - 18:09:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 177 : avg iteration time 49.84 seconds +[Apr 25, 2026 - 18:09:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 177 : data loading time 0.00 +[Apr 25, 2026 - 18:09:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 177 : avg forward pass time 12.90 +[Apr 25, 2026 - 18:09:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 177 : backward pass time 11.52 +[Apr 25, 2026 - 18:09:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 177 : optimizer step time 0.97 +[Apr 25, 2026 - 18:09:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 177-------------------- +[Apr 25, 2026 - 18:09:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0256 iter count: 1.0 +[Apr 25, 2026 - 18:09:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0256 iter count: 1.0 +[Apr 25, 2026 - 18:09:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:09:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:10:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093443 43.158855 43.003944 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.333496 70.525391 70.269531 +util 95.500000 99.000000 92.000000 +[Apr 25, 2026 - 18:10:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 178 : avg iteration time 46.94 seconds +[Apr 25, 2026 - 18:10:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 178 : data loading time 0.00 +[Apr 25, 2026 - 18:10:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 178 : avg forward pass time 11.44 +[Apr 25, 2026 - 18:10:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 178 : backward pass time 11.54 +[Apr 25, 2026 - 18:10:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 178 : optimizer step time 0.99 +[Apr 25, 2026 - 18:10:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 178-------------------- +[Apr 25, 2026 - 18:10:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0244 iter count: 1.0 +[Apr 25, 2026 - 18:10:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0244 iter count: 1.0 +[Apr 25, 2026 - 18:10:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:10:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:11:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093297 43.158260 43.003944 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.313965 70.564453 70.230469 +util 96.000000 100.000000 92.000000 +[Apr 25, 2026 - 18:11:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 179 : avg iteration time 53.01 seconds +[Apr 25, 2026 - 18:11:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 179 : data loading time 0.00 +[Apr 25, 2026 - 18:11:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 179 : avg forward pass time 14.48 +[Apr 25, 2026 - 18:11:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 179 : backward pass time 11.54 +[Apr 25, 2026 - 18:11:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 179 : optimizer step time 0.98 +[Apr 25, 2026 - 18:11:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 179-------------------- +[Apr 25, 2026 - 18:11:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0239 iter count: 1.0 +[Apr 25, 2026 - 18:11:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0239 iter count: 1.0 +[Apr 25, 2026 - 18:11:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:11:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093159 43.157131 43.004082 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.444336 135.458984 135.439453 +util 68.500000 77.000000 61.000000 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 180 : avg iteration time 175.37 seconds +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 180 : data loading time 0.00 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 180 : avg forward pass time 78.42 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 180 : backward pass time 9.15 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 180 : optimizer step time 0.34 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 180-------------------- +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 51101.3248 iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6231 iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 51101.3248iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1240iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6064 iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.4090 iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 157879.3281iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.3631 iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2724854.0000iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6231iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.2923 iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 723.8728iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.1237 iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 3817.4187iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.4345 iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 130.1849iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.3700 iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 46162.2539iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0242 iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:14:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 18:15:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093291 43.157639 43.004082 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.230957 70.525391 70.132812 +util 93.750000 99.000000 90.000000 +[Apr 25, 2026 - 18:15:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 181 : avg iteration time 51.42 seconds +[Apr 25, 2026 - 18:15:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 181 : data loading time 0.00 +[Apr 25, 2026 - 18:15:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 181 : avg forward pass time 13.67 +[Apr 25, 2026 - 18:15:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 181 : backward pass time 11.54 +[Apr 25, 2026 - 18:15:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 181 : optimizer step time 1.00 +[Apr 25, 2026 - 18:15:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 181-------------------- +[Apr 25, 2026 - 18:15:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0228 iter count: 1.0 +[Apr 25, 2026 - 18:15:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0228 iter count: 1.0 +[Apr 25, 2026 - 18:15:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:15:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:15:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093411 43.158142 43.004082 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.655762 70.701172 70.640625 +util 97.500000 99.000000 93.000000 +[Apr 25, 2026 - 18:15:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 182 : avg iteration time 49.90 seconds +[Apr 25, 2026 - 18:15:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 182 : data loading time 0.00 +[Apr 25, 2026 - 18:15:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 182 : avg forward pass time 12.92 +[Apr 25, 2026 - 18:15:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 182 : backward pass time 11.53 +[Apr 25, 2026 - 18:15:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 182 : optimizer step time 0.99 +[Apr 25, 2026 - 18:15:57 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 182-------------------- +[Apr 25, 2026 - 18:15:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0202 iter count: 1.0 +[Apr 25, 2026 - 18:15:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0202 iter count: 1.0 +[Apr 25, 2026 - 18:15:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:15:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:16:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093469 43.158371 43.004059 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.348145 70.701172 70.230469 +util 98.250000 100.000000 95.000000 +[Apr 25, 2026 - 18:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 183 : avg iteration time 52.83 seconds +[Apr 25, 2026 - 18:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 183 : data loading time 0.00 +[Apr 25, 2026 - 18:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 183 : avg forward pass time 14.39 +[Apr 25, 2026 - 18:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 183 : backward pass time 11.55 +[Apr 25, 2026 - 18:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 183 : optimizer step time 0.97 +[Apr 25, 2026 - 18:16:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 183-------------------- +[Apr 25, 2026 - 18:16:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0198 iter count: 1.0 +[Apr 25, 2026 - 18:16:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0198 iter count: 1.0 +[Apr 25, 2026 - 18:16:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:16:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:17:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093523 43.158611 43.004059 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.274902 70.701172 70.132812 +util 98.750000 100.000000 97.000000 +[Apr 25, 2026 - 18:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 184 : avg iteration time 46.94 seconds +[Apr 25, 2026 - 18:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 184 : data loading time 0.00 +[Apr 25, 2026 - 18:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 184 : avg forward pass time 11.45 +[Apr 25, 2026 - 18:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 184 : backward pass time 11.52 +[Apr 25, 2026 - 18:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 184 : optimizer step time 0.99 +[Apr 25, 2026 - 18:17:37 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 184-------------------- +[Apr 25, 2026 - 18:17:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0225 iter count: 1.0 +[Apr 25, 2026 - 18:17:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0225 iter count: 1.0 +[Apr 25, 2026 - 18:17:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:17:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093369 43.157948 43.004082 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.508301 135.517578 135.480469 +util 67.750000 76.000000 60.000000 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 185 : avg iteration time 169.91 seconds +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 185 : data loading time 0.00 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 185 : avg forward pass time 75.68 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 185 : backward pass time 9.15 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 185 : optimizer step time 0.34 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 185-------------------- +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 99021.9592 iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5561 iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 99021.9592iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1603iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.2214 iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.4793 iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 165177.5938iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 11.5056iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 3136679.7500iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5561iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.3608 iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 6134.0532iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.9007 iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2444.0286iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.0417 iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 438.5531iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.4882 iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 144.9558iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0222 iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:20:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 18:21:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093357 43.157948 43.004059 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.514160 70.544922 70.503906 +util 97.250000 99.000000 95.000000 +[Apr 25, 2026 - 18:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 186 : avg iteration time 54.27 seconds +[Apr 25, 2026 - 18:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 186 : data loading time 0.00 +[Apr 25, 2026 - 18:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 186 : avg forward pass time 15.11 +[Apr 25, 2026 - 18:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 186 : backward pass time 11.53 +[Apr 25, 2026 - 18:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 186 : optimizer step time 1.00 +[Apr 25, 2026 - 18:21:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 186-------------------- +[Apr 25, 2026 - 18:21:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0214 iter count: 1.0 +[Apr 25, 2026 - 18:21:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0214 iter count: 1.0 +[Apr 25, 2026 - 18:21:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:21:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:22:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093363 43.157948 43.004059 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.621582 70.681641 70.601562 +util 96.250000 99.000000 93.000000 +[Apr 25, 2026 - 18:22:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 187 : avg iteration time 46.98 seconds +[Apr 25, 2026 - 18:22:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 187 : data loading time 0.00 +[Apr 25, 2026 - 18:22:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 187 : avg forward pass time 11.48 +[Apr 25, 2026 - 18:22:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 187 : backward pass time 11.53 +[Apr 25, 2026 - 18:22:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 187 : optimizer step time 0.97 +[Apr 25, 2026 - 18:22:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 187-------------------- +[Apr 25, 2026 - 18:22:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0206 iter count: 1.0 +[Apr 25, 2026 - 18:22:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0206 iter count: 1.0 +[Apr 25, 2026 - 18:22:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:22:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:22:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093368 43.157948 43.004059 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.299316 70.564453 70.210938 +util 95.000000 99.000000 91.000000 +[Apr 25, 2026 - 18:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 188 : avg iteration time 46.98 seconds +[Apr 25, 2026 - 18:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 188 : data loading time 0.00 +[Apr 25, 2026 - 18:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 188 : avg forward pass time 11.46 +[Apr 25, 2026 - 18:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 188 : backward pass time 11.55 +[Apr 25, 2026 - 18:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 188 : optimizer step time 0.99 +[Apr 25, 2026 - 18:22:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 188-------------------- +[Apr 25, 2026 - 18:22:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0243 iter count: 1.0 +[Apr 25, 2026 - 18:22:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0243 iter count: 1.0 +[Apr 25, 2026 - 18:22:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:22:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:23:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093488 43.158440 43.004066 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.372559 70.681641 70.269531 +util 97.000000 99.000000 93.000000 +[Apr 25, 2026 - 18:23:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 189 : avg iteration time 52.95 seconds +[Apr 25, 2026 - 18:23:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 189 : data loading time 0.00 +[Apr 25, 2026 - 18:23:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 189 : avg forward pass time 14.46 +[Apr 25, 2026 - 18:23:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 189 : backward pass time 11.53 +[Apr 25, 2026 - 18:23:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 189 : optimizer step time 0.97 +[Apr 25, 2026 - 18:23:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 189-------------------- +[Apr 25, 2026 - 18:23:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0208 iter count: 1.0 +[Apr 25, 2026 - 18:23:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0208 iter count: 1.0 +[Apr 25, 2026 - 18:23:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:23:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093266 43.157555 43.004063 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.483887 135.500000 135.478516 +util 71.000000 76.000000 67.000000 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 190 : avg iteration time 175.51 seconds +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 190 : data loading time 0.00 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 190 : avg forward pass time 78.50 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 190 : backward pass time 9.13 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 190 : optimizer step time 0.34 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 190-------------------- +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 458842.4805 iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5584 iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 458842.4805iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.8959iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.9971 iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.2689 iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 642392.3750iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 54.2849iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 15213663.0000iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5584iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.2016 iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 32964.0703iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.0888 iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 481.8603iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.5438 iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 161.9964iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.4949 iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 59271.2500iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0205 iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:26:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 18:27:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093264 43.157555 43.004074 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.523926 70.525391 70.523438 +util 97.250000 100.000000 94.000000 +[Apr 25, 2026 - 18:27:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 191 : avg iteration time 54.18 seconds +[Apr 25, 2026 - 18:27:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 191 : data loading time 0.00 +[Apr 25, 2026 - 18:27:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 191 : avg forward pass time 15.07 +[Apr 25, 2026 - 18:27:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 191 : backward pass time 11.53 +[Apr 25, 2026 - 18:27:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 191 : optimizer step time 0.96 +[Apr 25, 2026 - 18:27:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 191-------------------- +[Apr 25, 2026 - 18:27:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0237 iter count: 1.0 +[Apr 25, 2026 - 18:27:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0237 iter count: 1.0 +[Apr 25, 2026 - 18:27:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:27:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:28:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093283 43.157562 43.004074 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.499512 70.503906 70.486328 +util 96.500000 99.000000 93.000000 +[Apr 25, 2026 - 18:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 192 : avg iteration time 46.99 seconds +[Apr 25, 2026 - 18:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 192 : data loading time 0.00 +[Apr 25, 2026 - 18:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 192 : avg forward pass time 11.48 +[Apr 25, 2026 - 18:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 192 : backward pass time 11.51 +[Apr 25, 2026 - 18:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 192 : optimizer step time 0.99 +[Apr 25, 2026 - 18:28:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 192-------------------- +[Apr 25, 2026 - 18:28:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0223 iter count: 1.0 +[Apr 25, 2026 - 18:28:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0223 iter count: 1.0 +[Apr 25, 2026 - 18:28:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:28:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:29:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093274 43.157578 43.004074 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.519043 70.564453 70.503906 +util 95.000000 99.000000 91.000000 +[Apr 25, 2026 - 18:29:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 193 : avg iteration time 52.78 seconds +[Apr 25, 2026 - 18:29:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 193 : data loading time 0.00 +[Apr 25, 2026 - 18:29:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 193 : avg forward pass time 14.36 +[Apr 25, 2026 - 18:29:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 193 : backward pass time 11.54 +[Apr 25, 2026 - 18:29:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 193 : optimizer step time 0.98 +[Apr 25, 2026 - 18:29:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 193-------------------- +[Apr 25, 2026 - 18:29:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0193 iter count: 1.0 +[Apr 25, 2026 - 18:29:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0193 iter count: 1.0 +[Apr 25, 2026 - 18:29:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:29:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:30:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093332 43.157814 43.004074 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.504395 70.505859 70.503906 +util 98.500000 99.000000 97.000000 +[Apr 25, 2026 - 18:30:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 194 : avg iteration time 52.87 seconds +[Apr 25, 2026 - 18:30:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 194 : data loading time 0.00 +[Apr 25, 2026 - 18:30:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 194 : avg forward pass time 14.40 +[Apr 25, 2026 - 18:30:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 194 : backward pass time 11.53 +[Apr 25, 2026 - 18:30:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 194 : optimizer step time 0.99 +[Apr 25, 2026 - 18:30:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 194-------------------- +[Apr 25, 2026 - 18:30:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0188 iter count: 1.0 +[Apr 25, 2026 - 18:30:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0188 iter count: 1.0 +[Apr 25, 2026 - 18:30:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:30:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093390 43.158039 43.004074 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.521484 135.535156 135.480469 +util 69.750000 75.000000 63.000000 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 195 : avg iteration time 166.88 seconds +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 195 : data loading time 0.00 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 195 : avg forward pass time 74.17 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 195 : backward pass time 9.15 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 195 : optimizer step time 0.34 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 195-------------------- +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 131485.8682 iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5233 iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 131485.8682iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5824iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7440 iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.8350 iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 335715.6562iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.4281 iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 6388668.0000iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5233iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.2355 iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 646.1459iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.0131 iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 414.1475iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.1044 iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 27138.6836iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 7.8350 iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 6388668.0000iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0213 iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 18:33:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093132 43.157028 43.004074 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.294434 70.896484 70.093750 +util 96.750000 100.000000 93.000000 +[Apr 25, 2026 - 18:33:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 196 : avg iteration time 54.27 seconds +[Apr 25, 2026 - 18:33:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 196 : data loading time 0.00 +[Apr 25, 2026 - 18:33:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 196 : avg forward pass time 15.11 +[Apr 25, 2026 - 18:33:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 196 : backward pass time 11.52 +[Apr 25, 2026 - 18:33:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 196 : optimizer step time 0.98 +[Apr 25, 2026 - 18:33:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 196-------------------- +[Apr 25, 2026 - 18:33:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0195 iter count: 1.0 +[Apr 25, 2026 - 18:33:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0195 iter count: 1.0 +[Apr 25, 2026 - 18:33:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:33:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:34:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093445 43.158253 43.004078 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.494629 70.583984 70.464844 +util 97.750000 100.000000 95.000000 +[Apr 25, 2026 - 18:34:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 197 : avg iteration time 52.83 seconds +[Apr 25, 2026 - 18:34:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 197 : data loading time 0.00 +[Apr 25, 2026 - 18:34:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 197 : avg forward pass time 14.39 +[Apr 25, 2026 - 18:34:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 197 : backward pass time 11.53 +[Apr 25, 2026 - 18:34:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 197 : optimizer step time 0.98 +[Apr 25, 2026 - 18:34:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 197-------------------- +[Apr 25, 2026 - 18:34:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0205 iter count: 1.0 +[Apr 25, 2026 - 18:34:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0205 iter count: 1.0 +[Apr 25, 2026 - 18:34:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:34:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:35:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093763 43.159554 43.004074 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.323730 70.544922 70.250000 +util 96.250000 100.000000 91.000000 +[Apr 25, 2026 - 18:35:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 198 : avg iteration time 49.88 seconds +[Apr 25, 2026 - 18:35:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 198 : data loading time 0.00 +[Apr 25, 2026 - 18:35:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 198 : avg forward pass time 12.90 +[Apr 25, 2026 - 18:35:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 198 : backward pass time 11.51 +[Apr 25, 2026 - 18:35:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 198 : optimizer step time 0.99 +[Apr 25, 2026 - 18:35:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 198-------------------- +[Apr 25, 2026 - 18:35:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0206 iter count: 1.0 +[Apr 25, 2026 - 18:35:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0206 iter count: 1.0 +[Apr 25, 2026 - 18:35:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:35:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:36:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093825 43.159801 43.004074 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.333496 70.701172 70.210938 +util 98.250000 100.000000 96.000000 +[Apr 25, 2026 - 18:36:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 199 : avg iteration time 49.92 seconds +[Apr 25, 2026 - 18:36:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 199 : data loading time 0.00 +[Apr 25, 2026 - 18:36:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 199 : avg forward pass time 12.93 +[Apr 25, 2026 - 18:36:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 199 : backward pass time 11.55 +[Apr 25, 2026 - 18:36:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 199 : optimizer step time 0.98 +[Apr 25, 2026 - 18:36:24 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 199-------------------- +[Apr 25, 2026 - 18:36:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0209 iter count: 1.0 +[Apr 25, 2026 - 18:36:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0209 iter count: 1.0 +[Apr 25, 2026 - 18:36:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:36:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093300 43.157665 43.004093 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.542480 135.576172 135.441406 +util 69.000000 77.000000 62.000000 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 200 : avg iteration time 175.94 seconds +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 200 : data loading time 0.00 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 200 : avg forward pass time 78.69 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 200 : backward pass time 9.15 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 200 : optimizer step time 0.34 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 200-------------------- +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 34616.0269 iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5634 iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 34616.0269iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5113iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7769 iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0215 iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 79017.6875iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.7294 iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1255330.7500iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5634iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.6624 iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1517.3379iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.9149 iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 340.2918iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.6073 iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 10042.4912iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.3478 iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 808.7963iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0208 iter count: 1.0 +[Apr 25, 2026 - 18:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 25, 2026 - 18:39:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 15.58 seconds +[Apr 25, 2026 - 18:39:36 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 200 +[Apr 25, 2026 - 18:39:36 | INFO | fastgen.utils.checkpointer:save:301 ] Saving FSDP model to prefix /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000200 +[Apr 25, 2026 - 18:39:50 | SUCCESS | fastgen.utils.checkpointer:save:329 ] Model saved at iteration 200 +[Apr 25, 2026 - 18:39:50 | INFO | fastgen.trainer:validate:358 ] Validation iteration 200 +[Apr 25, 2026 - 18:39:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 18:39:52 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 2.3s +[Apr 25, 2026 - 18:39:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 18:39:58 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 18:40:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 18:40:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 18:40:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 18:40:08 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 18:40:11 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 18:40:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 18:40:17 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 18:40:18 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 18:40:22 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 18:40:24 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 18:40:27 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 18:40:29 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 18:40:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 18:40:34 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 18:40:37 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 18:40:39 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 18:40:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging val0 stats at iteration 200-------------------- +[Apr 25, 2026 - 18:40:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 25, 2026 - 18:40:59 | INFO | fastgen.callbacks.wandb:on_validation_end:683 ] Logged 10 val videos at iteration 200 +[Apr 25, 2026 - 18:41:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093509 43.159966 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.772949 70.994141 70.699219 +util 98.000000 99.000000 95.000000 +[Apr 25, 2026 - 18:41:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 201 : avg iteration time 147.54 seconds +[Apr 25, 2026 - 18:41:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 201 : data loading time 0.00 +[Apr 25, 2026 - 18:41:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 201 : avg forward pass time 12.53 +[Apr 25, 2026 - 18:41:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 201 : backward pass time 11.55 +[Apr 25, 2026 - 18:41:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 201 : optimizer step time 0.98 +[Apr 25, 2026 - 18:41:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 201-------------------- +[Apr 25, 2026 - 18:41:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0180 iter count: 1.0 +[Apr 25, 2026 - 18:41:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0180 iter count: 1.0 +[Apr 25, 2026 - 18:41:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:41:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:42:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093515 43.159966 43.003475 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.021973 71.169922 70.972656 +util 95.500000 99.000000 90.000000 +[Apr 25, 2026 - 18:42:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 202 : avg iteration time 53.04 seconds +[Apr 25, 2026 - 18:42:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 202 : data loading time 0.00 +[Apr 25, 2026 - 18:42:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 202 : avg forward pass time 14.44 +[Apr 25, 2026 - 18:42:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 202 : backward pass time 11.58 +[Apr 25, 2026 - 18:42:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 202 : optimizer step time 0.99 +[Apr 25, 2026 - 18:42:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 202-------------------- +[Apr 25, 2026 - 18:42:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0206 iter count: 1.0 +[Apr 25, 2026 - 18:42:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0206 iter count: 1.0 +[Apr 25, 2026 - 18:42:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:42:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:43:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093514 43.159966 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.724121 71.033203 70.621094 +util 95.500000 100.000000 92.000000 +[Apr 25, 2026 - 18:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 203 : avg iteration time 50.03 seconds +[Apr 25, 2026 - 18:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 203 : data loading time 0.00 +[Apr 25, 2026 - 18:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 203 : avg forward pass time 12.97 +[Apr 25, 2026 - 18:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 203 : backward pass time 11.54 +[Apr 25, 2026 - 18:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 203 : optimizer step time 0.97 +[Apr 25, 2026 - 18:43:31 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 203-------------------- +[Apr 25, 2026 - 18:43:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0191 iter count: 1.0 +[Apr 25, 2026 - 18:43:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0191 iter count: 1.0 +[Apr 25, 2026 - 18:43:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:43:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:44:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093348 43.159321 43.003460 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.919434 71.169922 70.835938 +util 97.750000 99.000000 96.000000 +[Apr 25, 2026 - 18:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 204 : avg iteration time 47.05 seconds +[Apr 25, 2026 - 18:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 204 : data loading time 0.00 +[Apr 25, 2026 - 18:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 204 : avg forward pass time 11.48 +[Apr 25, 2026 - 18:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 204 : backward pass time 11.54 +[Apr 25, 2026 - 18:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 204 : optimizer step time 0.98 +[Apr 25, 2026 - 18:44:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 204-------------------- +[Apr 25, 2026 - 18:44:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0181 iter count: 1.0 +[Apr 25, 2026 - 18:44:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0181 iter count: 1.0 +[Apr 25, 2026 - 18:44:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:44:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093165 43.157242 43.003910 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.526367 135.535156 135.500000 +util 66.250000 71.000000 62.000000 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 205 : avg iteration time 172.95 seconds +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 205 : data loading time 0.00 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 205 : avg forward pass time 77.20 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 205 : backward pass time 9.17 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 205 : optimizer step time 0.34 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 205-------------------- +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 413753.9268 iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4909 iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 413753.9268iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.2108iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.9521 iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.2859 iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 563373.7500iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 6.7145 iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 15740308.0000iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4909iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.5556 iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 66912.2578iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.3067 iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 5505.3931iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.7582 iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 13580.4180iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.1992 iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 242399.4219iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0196 iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:47:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 18:48:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093307 43.157742 43.003929 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.772949 70.835938 70.583984 +util 97.250000 99.000000 93.000000 +[Apr 25, 2026 - 18:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 206 : avg iteration time 51.35 seconds +[Apr 25, 2026 - 18:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 206 : data loading time 0.00 +[Apr 25, 2026 - 18:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 206 : avg forward pass time 13.64 +[Apr 25, 2026 - 18:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 206 : backward pass time 11.56 +[Apr 25, 2026 - 18:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 206 : optimizer step time 0.98 +[Apr 25, 2026 - 18:48:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 206-------------------- +[Apr 25, 2026 - 18:48:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0211 iter count: 1.0 +[Apr 25, 2026 - 18:48:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0211 iter count: 1.0 +[Apr 25, 2026 - 18:48:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:48:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:48:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093287 43.157715 43.003925 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.680176 70.718750 70.564453 +util 98.000000 100.000000 94.000000 +[Apr 25, 2026 - 18:48:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 207 : avg iteration time 49.96 seconds +[Apr 25, 2026 - 18:48:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 207 : data loading time 0.00 +[Apr 25, 2026 - 18:48:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 207 : avg forward pass time 12.96 +[Apr 25, 2026 - 18:48:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 207 : backward pass time 11.53 +[Apr 25, 2026 - 18:48:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 207 : optimizer step time 0.98 +[Apr 25, 2026 - 18:48:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 207-------------------- +[Apr 25, 2026 - 18:48:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0217 iter count: 1.0 +[Apr 25, 2026 - 18:48:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0217 iter count: 1.0 +[Apr 25, 2026 - 18:48:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:48:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:49:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093289 43.157715 43.003933 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.758301 70.816406 70.583984 +util 98.750000 100.000000 97.000000 +[Apr 25, 2026 - 18:49:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 208 : avg iteration time 50.14 seconds +[Apr 25, 2026 - 18:49:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 208 : data loading time 0.00 +[Apr 25, 2026 - 18:49:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 208 : avg forward pass time 13.04 +[Apr 25, 2026 - 18:49:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 208 : backward pass time 11.54 +[Apr 25, 2026 - 18:49:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 208 : optimizer step time 0.99 +[Apr 25, 2026 - 18:49:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 208-------------------- +[Apr 25, 2026 - 18:49:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0206 iter count: 1.0 +[Apr 25, 2026 - 18:49:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0206 iter count: 1.0 +[Apr 25, 2026 - 18:49:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:49:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:50:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093351 43.157959 43.003914 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.807129 70.894531 70.544922 +util 98.250000 100.000000 96.000000 +[Apr 25, 2026 - 18:50:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 209 : avg iteration time 52.79 seconds +[Apr 25, 2026 - 18:50:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 209 : data loading time 0.00 +[Apr 25, 2026 - 18:50:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 209 : avg forward pass time 14.39 +[Apr 25, 2026 - 18:50:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 209 : backward pass time 11.54 +[Apr 25, 2026 - 18:50:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 209 : optimizer step time 0.98 +[Apr 25, 2026 - 18:50:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 209-------------------- +[Apr 25, 2026 - 18:50:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0199 iter count: 1.0 +[Apr 25, 2026 - 18:50:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0199 iter count: 1.0 +[Apr 25, 2026 - 18:50:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:50:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092962 43.157391 43.003910 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.493652 135.498047 135.480469 +util 71.250000 77.000000 67.000000 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 210 : avg iteration time 169.61 seconds +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 210 : data loading time 0.00 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 210 : avg forward pass time 75.54 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 210 : backward pass time 9.13 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 210 : optimizer step time 0.34 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 210-------------------- +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 135509.9709 iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5091 iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 135509.9709iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4994iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.9651 iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.7673 iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 319372.3125iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 6.8903 iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 5579250.5000iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5091iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.8124 iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 2048.4641iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.0134 iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 3062.0334iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.7943 iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 107866.9297iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.2238 iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 85.4293 iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0195 iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:53:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 18:54:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093034 43.157631 43.003910 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.582520 70.621094 70.466797 +util 97.500000 99.000000 93.000000 +[Apr 25, 2026 - 18:54:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 211 : avg iteration time 51.24 seconds +[Apr 25, 2026 - 18:54:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 211 : data loading time 0.00 +[Apr 25, 2026 - 18:54:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 211 : avg forward pass time 13.60 +[Apr 25, 2026 - 18:54:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 211 : backward pass time 11.52 +[Apr 25, 2026 - 18:54:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 211 : optimizer step time 0.98 +[Apr 25, 2026 - 18:54:16 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 211-------------------- +[Apr 25, 2026 - 18:54:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0191 iter count: 1.0 +[Apr 25, 2026 - 18:54:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0191 iter count: 1.0 +[Apr 25, 2026 - 18:54:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:54:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:55:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093022 43.157631 43.003910 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.592285 70.681641 70.562500 +util 95.750000 99.000000 91.000000 +[Apr 25, 2026 - 18:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 212 : avg iteration time 52.86 seconds +[Apr 25, 2026 - 18:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 212 : data loading time 0.00 +[Apr 25, 2026 - 18:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 212 : avg forward pass time 14.41 +[Apr 25, 2026 - 18:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 212 : backward pass time 11.54 +[Apr 25, 2026 - 18:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 212 : optimizer step time 0.97 +[Apr 25, 2026 - 18:55:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 212-------------------- +[Apr 25, 2026 - 18:55:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0201 iter count: 1.0 +[Apr 25, 2026 - 18:55:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0201 iter count: 1.0 +[Apr 25, 2026 - 18:55:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:55:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:55:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093022 43.157631 43.003910 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.548340 70.564453 70.542969 +util 95.000000 99.000000 91.000000 +[Apr 25, 2026 - 18:55:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 213 : avg iteration time 47.03 seconds +[Apr 25, 2026 - 18:55:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 213 : data loading time 0.00 +[Apr 25, 2026 - 18:55:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 213 : avg forward pass time 11.49 +[Apr 25, 2026 - 18:55:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 213 : backward pass time 11.53 +[Apr 25, 2026 - 18:55:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 213 : optimizer step time 0.98 +[Apr 25, 2026 - 18:55:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 213-------------------- +[Apr 25, 2026 - 18:55:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0228 iter count: 1.0 +[Apr 25, 2026 - 18:55:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0228 iter count: 1.0 +[Apr 25, 2026 - 18:55:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:55:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:56:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093034 43.157631 43.003910 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.543457 70.582031 70.427734 +util 96.000000 100.000000 92.000000 +[Apr 25, 2026 - 18:56:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 214 : avg iteration time 52.82 seconds +[Apr 25, 2026 - 18:56:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 214 : data loading time 0.00 +[Apr 25, 2026 - 18:56:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 214 : avg forward pass time 14.39 +[Apr 25, 2026 - 18:56:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 214 : backward pass time 11.53 +[Apr 25, 2026 - 18:56:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 214 : optimizer step time 0.99 +[Apr 25, 2026 - 18:56:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 214-------------------- +[Apr 25, 2026 - 18:56:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0203 iter count: 1.0 +[Apr 25, 2026 - 18:56:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0203 iter count: 1.0 +[Apr 25, 2026 - 18:56:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:56:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093028 43.157639 43.003922 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.542480 135.556641 135.500000 +util 70.500000 76.000000 66.000000 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 215 : avg iteration time 172.65 seconds +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 215 : data loading time 0.00 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 215 : avg forward pass time 77.04 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 215 : backward pass time 9.13 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 215 : optimizer step time 0.34 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 215-------------------- +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 76062.2014 iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6301 iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 76062.2014iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.8758iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.5429 iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.9089 iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 109681.1719iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 21.8842iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1002322.1875iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6301iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.7604 iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 13641.7891iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.7658 iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 13789.5557iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.3114 iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 101.7855iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.8754 iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 126845.4688iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0203 iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 19:00:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093022 43.157631 43.003910 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.724121 70.757812 70.623047 +util 96.250000 99.000000 91.000000 +[Apr 25, 2026 - 19:00:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 216 : avg iteration time 51.28 seconds +[Apr 25, 2026 - 19:00:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 216 : data loading time 0.00 +[Apr 25, 2026 - 19:00:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 216 : avg forward pass time 13.62 +[Apr 25, 2026 - 19:00:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 216 : backward pass time 11.53 +[Apr 25, 2026 - 19:00:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 216 : optimizer step time 0.99 +[Apr 25, 2026 - 19:00:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 216-------------------- +[Apr 25, 2026 - 19:00:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0180 iter count: 1.0 +[Apr 25, 2026 - 19:00:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0180 iter count: 1.0 +[Apr 25, 2026 - 19:00:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:00:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:01:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093028 43.157631 43.003910 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.616699 70.662109 70.601562 +util 96.000000 99.000000 93.000000 +[Apr 25, 2026 - 19:01:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 217 : avg iteration time 49.88 seconds +[Apr 25, 2026 - 19:01:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 217 : data loading time 0.00 +[Apr 25, 2026 - 19:01:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 217 : avg forward pass time 12.91 +[Apr 25, 2026 - 19:01:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 217 : backward pass time 11.53 +[Apr 25, 2026 - 19:01:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 217 : optimizer step time 0.99 +[Apr 25, 2026 - 19:01:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 217-------------------- +[Apr 25, 2026 - 19:01:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0222 iter count: 1.0 +[Apr 25, 2026 - 19:01:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0222 iter count: 1.0 +[Apr 25, 2026 - 19:01:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:01:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:02:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093028 43.157631 43.003910 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.621582 70.681641 70.601562 +util 98.000000 99.000000 96.000000 +[Apr 25, 2026 - 19:02:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 218 : avg iteration time 52.83 seconds +[Apr 25, 2026 - 19:02:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 218 : data loading time 0.00 +[Apr 25, 2026 - 19:02:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 218 : avg forward pass time 14.38 +[Apr 25, 2026 - 19:02:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 218 : backward pass time 11.53 +[Apr 25, 2026 - 19:02:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 218 : optimizer step time 0.98 +[Apr 25, 2026 - 19:02:15 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 218-------------------- +[Apr 25, 2026 - 19:02:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0204 iter count: 1.0 +[Apr 25, 2026 - 19:02:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0204 iter count: 1.0 +[Apr 25, 2026 - 19:02:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:02:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 19:03:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093023 43.157631 43.003914 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.592285 70.601562 70.564453 +util 98.250000 100.000000 95.000000 +[Apr 25, 2026 - 19:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 219 : avg iteration time 49.88 seconds +[Apr 25, 2026 - 19:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 219 : data loading time 0.00 +[Apr 25, 2026 - 19:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 219 : avg forward pass time 12.93 +[Apr 25, 2026 - 19:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 219 : backward pass time 11.53 +[Apr 25, 2026 - 19:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 219 : optimizer step time 0.97 +[Apr 25, 2026 - 19:03:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 219-------------------- +[Apr 25, 2026 - 19:03:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0205 iter count: 1.0 +[Apr 25, 2026 - 19:03:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0205 iter count: 1.0 +[Apr 25, 2026 - 19:03:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:03:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092767 43.156555 43.003948 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.537598 135.597656 135.517578 +util 70.250000 76.000000 66.000000 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 220 : avg iteration time 172.72 seconds +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 220 : data loading time 0.00 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 220 : avg forward pass time 77.08 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 220 : backward pass time 9.14 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 220 : optimizer step time 0.34 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 220-------------------- +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 169502.1924 iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5447 iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 169502.1924iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 5.1658iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.3552 iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.3839 iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 283614.2188iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 111.0860iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2591573.0000iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5447iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.7979 iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 802808.8750iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.7176 iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 683693.1875iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.9097 iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 336.7387iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.8606 iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 123161.5859iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0222 iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:05:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 19:06:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092757 43.156551 43.003925 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.689941 70.757812 70.486328 +util 95.750000 99.000000 92.000000 +[Apr 25, 2026 - 19:06:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 221 : avg iteration time 48.37 seconds +[Apr 25, 2026 - 19:06:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 221 : data loading time 0.00 +[Apr 25, 2026 - 19:06:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 221 : avg forward pass time 12.15 +[Apr 25, 2026 - 19:06:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 221 : backward pass time 11.53 +[Apr 25, 2026 - 19:06:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 221 : optimizer step time 0.99 +[Apr 25, 2026 - 19:06:46 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 221-------------------- +[Apr 25, 2026 - 19:06:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0226 iter count: 1.0 +[Apr 25, 2026 - 19:06:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0226 iter count: 1.0 +[Apr 25, 2026 - 19:06:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:06:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:07:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093101 43.157940 43.003910 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.660645 70.718750 70.486328 +util 97.000000 99.000000 94.000000 +[Apr 25, 2026 - 19:07:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 222 : avg iteration time 52.82 seconds +[Apr 25, 2026 - 19:07:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 222 : data loading time 0.00 +[Apr 25, 2026 - 19:07:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 222 : avg forward pass time 14.38 +[Apr 25, 2026 - 19:07:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 222 : backward pass time 11.56 +[Apr 25, 2026 - 19:07:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 222 : optimizer step time 0.97 +[Apr 25, 2026 - 19:07:39 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 222-------------------- +[Apr 25, 2026 - 19:07:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0217 iter count: 1.0 +[Apr 25, 2026 - 19:07:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0217 iter count: 1.0 +[Apr 25, 2026 - 19:07:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:07:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:08:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093106 43.157959 43.003914 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.714355 70.757812 70.583984 +util 94.750000 99.000000 91.000000 +[Apr 25, 2026 - 19:08:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 223 : avg iteration time 52.97 seconds +[Apr 25, 2026 - 19:08:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 223 : data loading time 0.00 +[Apr 25, 2026 - 19:08:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 223 : avg forward pass time 14.42 +[Apr 25, 2026 - 19:08:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 223 : backward pass time 11.56 +[Apr 25, 2026 - 19:08:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 223 : optimizer step time 1.01 +[Apr 25, 2026 - 19:08:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 223-------------------- +[Apr 25, 2026 - 19:08:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0206 iter count: 1.0 +[Apr 25, 2026 - 19:08:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0206 iter count: 1.0 +[Apr 25, 2026 - 19:08:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:08:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:09:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093112 43.157963 43.003914 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.709473 70.757812 70.564453 +util 94.750000 99.000000 91.000000 +[Apr 25, 2026 - 19:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 224 : avg iteration time 46.99 seconds +[Apr 25, 2026 - 19:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 224 : data loading time 0.00 +[Apr 25, 2026 - 19:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 224 : avg forward pass time 11.47 +[Apr 25, 2026 - 19:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 224 : backward pass time 11.54 +[Apr 25, 2026 - 19:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 224 : optimizer step time 0.98 +[Apr 25, 2026 - 19:09:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 224-------------------- +[Apr 25, 2026 - 19:09:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0235 iter count: 1.0 +[Apr 25, 2026 - 19:09:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0235 iter count: 1.0 +[Apr 25, 2026 - 19:09:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:09:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092882 43.157051 43.003925 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.506348 135.537109 135.496094 +util 68.000000 76.000000 61.000000 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 225 : avg iteration time 178.38 seconds +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 225 : data loading time 0.00 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 225 : avg forward pass time 79.91 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 225 : backward pass time 9.14 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 225 : optimizer step time 0.34 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 225-------------------- +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 81486.1406 iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5321 iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 81486.1406iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.8601iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.0132 iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.8205 iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 148640.6094iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 56.0580iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 839795.7500iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5321iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.0132 iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 56.0580 iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.9636 iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 20481.8379iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.0748 iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 3461.9116iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.8108 iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 15087.5029iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0216 iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:12:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 19:13:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092996 43.157516 43.003914 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.631348 70.662109 70.621094 +util 97.000000 99.000000 94.000000 +[Apr 25, 2026 - 19:13:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 226 : avg iteration time 54.26 seconds +[Apr 25, 2026 - 19:13:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 226 : data loading time 0.00 +[Apr 25, 2026 - 19:13:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 226 : avg forward pass time 15.10 +[Apr 25, 2026 - 19:13:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 226 : backward pass time 11.55 +[Apr 25, 2026 - 19:13:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 226 : optimizer step time 0.99 +[Apr 25, 2026 - 19:13:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 226-------------------- +[Apr 25, 2026 - 19:13:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0186 iter count: 1.0 +[Apr 25, 2026 - 19:13:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0186 iter count: 1.0 +[Apr 25, 2026 - 19:13:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:13:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:14:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093107 43.157963 43.003914 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.782715 70.816406 70.681641 +util 97.750000 99.000000 96.000000 +[Apr 25, 2026 - 19:14:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 227 : avg iteration time 52.81 seconds +[Apr 25, 2026 - 19:14:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 227 : data loading time 0.00 +[Apr 25, 2026 - 19:14:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 227 : avg forward pass time 14.39 +[Apr 25, 2026 - 19:14:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 227 : backward pass time 11.53 +[Apr 25, 2026 - 19:14:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 227 : optimizer step time 0.98 +[Apr 25, 2026 - 19:14:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 227-------------------- +[Apr 25, 2026 - 19:14:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0194 iter count: 1.0 +[Apr 25, 2026 - 19:14:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0194 iter count: 1.0 +[Apr 25, 2026 - 19:14:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:14:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:14:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093113 43.157963 43.003929 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.641113 70.642578 70.640625 +util 98.500000 100.000000 96.000000 +[Apr 25, 2026 - 19:14:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 228 : avg iteration time 47.08 seconds +[Apr 25, 2026 - 19:14:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 228 : data loading time 0.00 +[Apr 25, 2026 - 19:14:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 228 : avg forward pass time 11.51 +[Apr 25, 2026 - 19:14:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 228 : backward pass time 11.53 +[Apr 25, 2026 - 19:14:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 228 : optimizer step time 0.99 +[Apr 25, 2026 - 19:14:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 228-------------------- +[Apr 25, 2026 - 19:14:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0215 iter count: 1.0 +[Apr 25, 2026 - 19:14:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0215 iter count: 1.0 +[Apr 25, 2026 - 19:14:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:14:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:15:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093175 43.158211 43.003914 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.680176 70.681641 70.679688 +util 97.500000 100.000000 93.000000 +[Apr 25, 2026 - 19:15:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 229 : avg iteration time 52.82 seconds +[Apr 25, 2026 - 19:15:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 229 : data loading time 0.00 +[Apr 25, 2026 - 19:15:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 229 : avg forward pass time 14.38 +[Apr 25, 2026 - 19:15:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 229 : backward pass time 11.53 +[Apr 25, 2026 - 19:15:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 229 : optimizer step time 0.97 +[Apr 25, 2026 - 19:15:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 229-------------------- +[Apr 25, 2026 - 19:15:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0164 iter count: 1.0 +[Apr 25, 2026 - 19:15:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0164 iter count: 1.0 +[Apr 25, 2026 - 19:15:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:15:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093047 43.157688 43.003937 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.547363 135.576172 135.460938 +util 69.250000 75.000000 65.000000 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 230 : avg iteration time 169.89 seconds +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 230 : data loading time 0.00 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 230 : avg forward pass time 75.67 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 230 : backward pass time 9.14 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 230 : optimizer step time 0.34 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 230-------------------- +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 175920.6318 iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5806 iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 175920.6318iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4858iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.7110 iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.6521 iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 241516.9062iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 30.6276iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 4431547.0000iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5806iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.2756 iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 700.1472iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.0729 iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 25481.8926iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.8547 iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 2229.3833iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.8286 iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 115512.3906iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0195 iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:18:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 19:19:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093040 43.157696 43.003914 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.636230 70.679688 70.505859 +util 98.250000 100.000000 95.000000 +[Apr 25, 2026 - 19:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 231 : avg iteration time 54.28 seconds +[Apr 25, 2026 - 19:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 231 : data loading time 0.00 +[Apr 25, 2026 - 19:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 231 : avg forward pass time 15.11 +[Apr 25, 2026 - 19:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 231 : backward pass time 11.55 +[Apr 25, 2026 - 19:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 231 : optimizer step time 0.97 +[Apr 25, 2026 - 19:19:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 231-------------------- +[Apr 25, 2026 - 19:19:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0183 iter count: 1.0 +[Apr 25, 2026 - 19:19:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0183 iter count: 1.0 +[Apr 25, 2026 - 19:19:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:19:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:20:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093040 43.157688 43.003922 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.655762 70.718750 70.466797 +util 96.250000 99.000000 93.000000 +[Apr 25, 2026 - 19:20:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 232 : avg iteration time 49.90 seconds +[Apr 25, 2026 - 19:20:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 232 : data loading time 0.00 +[Apr 25, 2026 - 19:20:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 232 : avg forward pass time 12.92 +[Apr 25, 2026 - 19:20:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 232 : backward pass time 11.55 +[Apr 25, 2026 - 19:20:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 232 : optimizer step time 0.98 +[Apr 25, 2026 - 19:20:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 232-------------------- +[Apr 25, 2026 - 19:20:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0190 iter count: 1.0 +[Apr 25, 2026 - 19:20:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0190 iter count: 1.0 +[Apr 25, 2026 - 19:20:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:20:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:21:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093044 43.157688 43.003937 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.611816 70.621094 70.583984 +util 96.500000 99.000000 91.000000 +[Apr 25, 2026 - 19:21:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 233 : avg iteration time 46.92 seconds +[Apr 25, 2026 - 19:21:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 233 : data loading time 0.00 +[Apr 25, 2026 - 19:21:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 233 : avg forward pass time 11.44 +[Apr 25, 2026 - 19:21:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 233 : backward pass time 11.53 +[Apr 25, 2026 - 19:21:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 233 : optimizer step time 0.97 +[Apr 25, 2026 - 19:21:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 233-------------------- +[Apr 25, 2026 - 19:21:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0186 iter count: 1.0 +[Apr 25, 2026 - 19:21:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0186 iter count: 1.0 +[Apr 25, 2026 - 19:21:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:21:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:21:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093074 43.157711 43.004032 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.680176 70.738281 70.505859 +util 96.750000 99.000000 91.000000 +[Apr 25, 2026 - 19:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 234 : avg iteration time 49.94 seconds +[Apr 25, 2026 - 19:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 234 : data loading time 0.00 +[Apr 25, 2026 - 19:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 234 : avg forward pass time 12.96 +[Apr 25, 2026 - 19:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 234 : backward pass time 11.52 +[Apr 25, 2026 - 19:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 234 : optimizer step time 0.97 +[Apr 25, 2026 - 19:21:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 234-------------------- +[Apr 25, 2026 - 19:21:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0169 iter count: 1.0 +[Apr 25, 2026 - 19:21:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0169 iter count: 1.0 +[Apr 25, 2026 - 19:21:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:21:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093074 43.157711 43.004032 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.550781 135.554688 135.539062 +util 69.500000 75.000000 65.000000 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 235 : avg iteration time 169.56 seconds +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 235 : data loading time 0.00 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 235 : avg forward pass time 75.50 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 235 : backward pass time 9.14 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 235 : optimizer step time 0.34 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 235-------------------- +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 198753.3428 iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5578 iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 198753.3428iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.7045iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7928 iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.1666 iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 530492.8750iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.8818 iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 12399966.0000iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5578iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.5857 iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 525089.8750iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.2719 iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 37940.7227iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.0851 iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 3534.3892iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.3905 iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 880.9191iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0185 iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:24:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 19:25:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093202 43.158184 43.004032 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.680176 70.699219 70.623047 +util 97.250000 99.000000 92.000000 +[Apr 25, 2026 - 19:25:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 236 : avg iteration time 48.45 seconds +[Apr 25, 2026 - 19:25:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 236 : data loading time 0.00 +[Apr 25, 2026 - 19:25:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 236 : avg forward pass time 12.21 +[Apr 25, 2026 - 19:25:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 236 : backward pass time 11.52 +[Apr 25, 2026 - 19:25:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 236 : optimizer step time 0.98 +[Apr 25, 2026 - 19:25:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 236-------------------- +[Apr 25, 2026 - 19:25:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0182 iter count: 1.0 +[Apr 25, 2026 - 19:25:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0182 iter count: 1.0 +[Apr 25, 2026 - 19:25:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:25:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:26:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093258 43.158421 43.004055 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.719238 70.738281 70.662109 +util 97.750000 100.000000 95.000000 +[Apr 25, 2026 - 19:26:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 237 : avg iteration time 46.95 seconds +[Apr 25, 2026 - 19:26:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 237 : data loading time 0.00 +[Apr 25, 2026 - 19:26:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 237 : avg forward pass time 11.45 +[Apr 25, 2026 - 19:26:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 237 : backward pass time 11.52 +[Apr 25, 2026 - 19:26:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 237 : optimizer step time 0.99 +[Apr 25, 2026 - 19:26:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 237-------------------- +[Apr 25, 2026 - 19:26:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0195 iter count: 1.0 +[Apr 25, 2026 - 19:26:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0195 iter count: 1.0 +[Apr 25, 2026 - 19:26:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:26:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:27:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093186 43.158161 43.004032 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.650879 70.681641 70.640625 +util 96.250000 100.000000 92.000000 +[Apr 25, 2026 - 19:27:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 238 : avg iteration time 52.78 seconds +[Apr 25, 2026 - 19:27:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 238 : data loading time 0.00 +[Apr 25, 2026 - 19:27:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 238 : avg forward pass time 14.38 +[Apr 25, 2026 - 19:27:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 238 : backward pass time 11.52 +[Apr 25, 2026 - 19:27:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 238 : optimizer step time 0.96 +[Apr 25, 2026 - 19:27:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 238-------------------- +[Apr 25, 2026 - 19:27:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0177 iter count: 1.0 +[Apr 25, 2026 - 19:27:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0177 iter count: 1.0 +[Apr 25, 2026 - 19:27:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:27:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:28:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093193 43.158180 43.004032 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.577637 70.601562 70.505859 +util 97.750000 100.000000 95.000000 +[Apr 25, 2026 - 19:28:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 239 : avg iteration time 52.83 seconds +[Apr 25, 2026 - 19:28:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 239 : data loading time 0.00 +[Apr 25, 2026 - 19:28:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 239 : avg forward pass time 14.40 +[Apr 25, 2026 - 19:28:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 239 : backward pass time 11.52 +[Apr 25, 2026 - 19:28:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 239 : optimizer step time 0.99 +[Apr 25, 2026 - 19:28:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 239-------------------- +[Apr 25, 2026 - 19:28:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0169 iter count: 1.0 +[Apr 25, 2026 - 19:28:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0169 iter count: 1.0 +[Apr 25, 2026 - 19:28:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:28:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093186 43.158161 43.004032 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.502441 135.517578 135.460938 +util 69.250000 77.000000 63.000000 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 240 : avg iteration time 173.14 seconds +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 240 : data loading time 0.00 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 240 : avg forward pass time 77.29 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 240 : backward pass time 9.15 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 240 : optimizer step time 0.34 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 240-------------------- +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 109811.8291 iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6257 iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 109811.8291iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5489iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.6771 iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.2565 iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 171414.6719iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 28.6204iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2008767.6250iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6257iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.3562 iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 44913.7227iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.9900 iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2921.9075iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.5135 iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 8325.2637iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.0319 iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 3177.0652iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0197 iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:30:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 19:31:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093253 43.158428 43.004032 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.645996 70.679688 70.544922 +util 98.250000 100.000000 95.000000 +[Apr 25, 2026 - 19:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 241 : avg iteration time 51.34 seconds +[Apr 25, 2026 - 19:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 241 : data loading time 0.00 +[Apr 25, 2026 - 19:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 241 : avg forward pass time 13.64 +[Apr 25, 2026 - 19:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 241 : backward pass time 11.52 +[Apr 25, 2026 - 19:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 241 : optimizer step time 0.99 +[Apr 25, 2026 - 19:31:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 241-------------------- +[Apr 25, 2026 - 19:31:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0216 iter count: 1.0 +[Apr 25, 2026 - 19:31:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0216 iter count: 1.0 +[Apr 25, 2026 - 19:31:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:31:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:32:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093247 43.158405 43.004032 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.631348 70.640625 70.603516 +util 97.500000 100.000000 92.000000 +[Apr 25, 2026 - 19:32:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 242 : avg iteration time 46.97 seconds +[Apr 25, 2026 - 19:32:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 242 : data loading time 0.00 +[Apr 25, 2026 - 19:32:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 242 : avg forward pass time 11.46 +[Apr 25, 2026 - 19:32:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 242 : backward pass time 11.51 +[Apr 25, 2026 - 19:32:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 242 : optimizer step time 1.00 +[Apr 25, 2026 - 19:32:37 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 242-------------------- +[Apr 25, 2026 - 19:32:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0189 iter count: 1.0 +[Apr 25, 2026 - 19:32:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0189 iter count: 1.0 +[Apr 25, 2026 - 19:32:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:32:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:33:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093222 43.158279 43.004032 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.631348 70.660156 70.544922 +util 97.750000 100.000000 93.000000 +[Apr 25, 2026 - 19:33:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 243 : avg iteration time 49.87 seconds +[Apr 25, 2026 - 19:33:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 243 : data loading time 0.00 +[Apr 25, 2026 - 19:33:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 243 : avg forward pass time 12.91 +[Apr 25, 2026 - 19:33:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 243 : backward pass time 11.54 +[Apr 25, 2026 - 19:33:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 243 : optimizer step time 0.98 +[Apr 25, 2026 - 19:33:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 243-------------------- +[Apr 25, 2026 - 19:33:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0185 iter count: 1.0 +[Apr 25, 2026 - 19:33:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0185 iter count: 1.0 +[Apr 25, 2026 - 19:33:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:33:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:34:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093222 43.158279 43.004032 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.558105 70.603516 70.542969 +util 97.250000 99.000000 92.000000 +[Apr 25, 2026 - 19:34:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 244 : avg iteration time 46.93 seconds +[Apr 25, 2026 - 19:34:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 244 : data loading time 0.00 +[Apr 25, 2026 - 19:34:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 244 : avg forward pass time 11.44 +[Apr 25, 2026 - 19:34:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 244 : backward pass time 11.53 +[Apr 25, 2026 - 19:34:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 244 : optimizer step time 0.98 +[Apr 25, 2026 - 19:34:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 244-------------------- +[Apr 25, 2026 - 19:34:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0197 iter count: 1.0 +[Apr 25, 2026 - 19:34:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0197 iter count: 1.0 +[Apr 25, 2026 - 19:34:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:34:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092937 43.157162 43.004032 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.463379 135.539062 135.437500 +util 72.750000 78.000000 65.000000 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 245 : avg iteration time 169.64 seconds +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 245 : data loading time 0.00 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 245 : avg forward pass time 75.56 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 245 : backward pass time 9.14 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 245 : optimizer step time 0.34 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 245-------------------- +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 228739.6729 iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5711 iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 228739.6729iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5499iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.8814 iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.1003 iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 465379.0938iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 43.0730iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 10859326.0000iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5711iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.3631 iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 6161.9385iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.4953 iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 8028.0728iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 7.4062 iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 2709948.0000iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.2661 iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 5075.6382iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0184 iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:37:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 19:37:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092949 43.157162 43.004036 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.567871 70.642578 70.542969 +util 98.500000 100.000000 96.000000 +[Apr 25, 2026 - 19:37:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 246 : avg iteration time 51.40 seconds +[Apr 25, 2026 - 19:37:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 246 : data loading time 0.00 +[Apr 25, 2026 - 19:37:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 246 : avg forward pass time 13.67 +[Apr 25, 2026 - 19:37:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 246 : backward pass time 11.52 +[Apr 25, 2026 - 19:37:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 246 : optimizer step time 0.99 +[Apr 25, 2026 - 19:37:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 246-------------------- +[Apr 25, 2026 - 19:37:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0180 iter count: 1.0 +[Apr 25, 2026 - 19:37:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0180 iter count: 1.0 +[Apr 25, 2026 - 19:37:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:37:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:38:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093128 43.157906 43.004055 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.655762 70.679688 70.583984 +util 98.500000 99.000000 97.000000 +[Apr 25, 2026 - 19:38:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 247 : avg iteration time 49.94 seconds +[Apr 25, 2026 - 19:38:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 247 : data loading time 0.00 +[Apr 25, 2026 - 19:38:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 247 : avg forward pass time 12.94 +[Apr 25, 2026 - 19:38:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 247 : backward pass time 11.54 +[Apr 25, 2026 - 19:38:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 247 : optimizer step time 0.99 +[Apr 25, 2026 - 19:38:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 247-------------------- +[Apr 25, 2026 - 19:38:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0190 iter count: 1.0 +[Apr 25, 2026 - 19:38:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0190 iter count: 1.0 +[Apr 25, 2026 - 19:38:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:38:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:39:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093504 43.159386 43.004051 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.577637 70.623047 70.562500 +util 97.000000 99.000000 92.000000 +[Apr 25, 2026 - 19:39:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 248 : avg iteration time 52.82 seconds +[Apr 25, 2026 - 19:39:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 248 : data loading time 0.00 +[Apr 25, 2026 - 19:39:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 248 : avg forward pass time 14.39 +[Apr 25, 2026 - 19:39:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 248 : backward pass time 11.53 +[Apr 25, 2026 - 19:39:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 248 : optimizer step time 0.99 +[Apr 25, 2026 - 19:39:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 248-------------------- +[Apr 25, 2026 - 19:39:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0179 iter count: 1.0 +[Apr 25, 2026 - 19:39:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0179 iter count: 1.0 +[Apr 25, 2026 - 19:39:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:39:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:40:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093499 43.159363 43.004036 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.665527 70.679688 70.623047 +util 96.250000 100.000000 92.000000 +[Apr 25, 2026 - 19:40:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 249 : avg iteration time 49.89 seconds +[Apr 25, 2026 - 19:40:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 249 : data loading time 0.00 +[Apr 25, 2026 - 19:40:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 249 : avg forward pass time 12.92 +[Apr 25, 2026 - 19:40:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 249 : backward pass time 11.52 +[Apr 25, 2026 - 19:40:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 249 : optimizer step time 1.00 +[Apr 25, 2026 - 19:40:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 249-------------------- +[Apr 25, 2026 - 19:40:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0190 iter count: 1.0 +[Apr 25, 2026 - 19:40:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0190 iter count: 1.0 +[Apr 25, 2026 - 19:40:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:40:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092897 43.157001 43.004036 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.552246 135.576172 135.480469 +util 63.750000 75.000000 54.000000 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 250 : avg iteration time 179.05 seconds +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 250 : data loading time 0.00 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 250 : avg forward pass time 80.19 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 250 : backward pass time 9.18 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 250 : optimizer step time 0.40 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 250-------------------- +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 718157.6694 iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5527 iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 718157.6694iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.8086iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.6162 iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.4714 iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 902433.1250iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 25.3386iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 22810030.0000iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5527iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.2344 iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 4763.9658iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.3366 iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 5843.7959iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.4135 iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 6815.6636iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.8676 iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 16903.2305iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0188 iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:43:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 19:44:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092967 43.157261 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.606934 70.660156 70.447266 +util 97.750000 100.000000 93.000000 +[Apr 25, 2026 - 19:44:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 251 : avg iteration time 54.27 seconds +[Apr 25, 2026 - 19:44:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 251 : data loading time 0.00 +[Apr 25, 2026 - 19:44:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 251 : avg forward pass time 15.11 +[Apr 25, 2026 - 19:44:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 251 : backward pass time 11.54 +[Apr 25, 2026 - 19:44:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 251 : optimizer step time 0.97 +[Apr 25, 2026 - 19:44:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 251-------------------- +[Apr 25, 2026 - 19:44:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0184 iter count: 1.0 +[Apr 25, 2026 - 19:44:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0184 iter count: 1.0 +[Apr 25, 2026 - 19:44:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:44:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:45:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093281 43.158493 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.626465 70.679688 70.466797 +util 97.250000 100.000000 92.000000 +[Apr 25, 2026 - 19:45:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 252 : avg iteration time 52.85 seconds +[Apr 25, 2026 - 19:45:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 252 : data loading time 0.00 +[Apr 25, 2026 - 19:45:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 252 : avg forward pass time 14.38 +[Apr 25, 2026 - 19:45:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 252 : backward pass time 11.54 +[Apr 25, 2026 - 19:45:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 252 : optimizer step time 0.99 +[Apr 25, 2026 - 19:45:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 252-------------------- +[Apr 25, 2026 - 19:45:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0206 iter count: 1.0 +[Apr 25, 2026 - 19:45:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0206 iter count: 1.0 +[Apr 25, 2026 - 19:45:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:45:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:46:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093285 43.158497 43.004063 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.421387 70.466797 70.406250 +util 97.750000 100.000000 93.000000 +[Apr 25, 2026 - 19:46:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 253 : avg iteration time 49.94 seconds +[Apr 25, 2026 - 19:46:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 253 : data loading time 0.00 +[Apr 25, 2026 - 19:46:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 253 : avg forward pass time 12.94 +[Apr 25, 2026 - 19:46:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 253 : backward pass time 11.52 +[Apr 25, 2026 - 19:46:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 253 : optimizer step time 0.99 +[Apr 25, 2026 - 19:46:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 253-------------------- +[Apr 25, 2026 - 19:46:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0178 iter count: 1.0 +[Apr 25, 2026 - 19:46:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0178 iter count: 1.0 +[Apr 25, 2026 - 19:46:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:46:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:46:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093246 43.158386 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.597168 70.640625 70.466797 +util 97.000000 99.000000 94.000000 +[Apr 25, 2026 - 19:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 254 : avg iteration time 49.94 seconds +[Apr 25, 2026 - 19:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 254 : data loading time 0.00 +[Apr 25, 2026 - 19:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 254 : avg forward pass time 12.95 +[Apr 25, 2026 - 19:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 254 : backward pass time 11.52 +[Apr 25, 2026 - 19:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 254 : optimizer step time 0.98 +[Apr 25, 2026 - 19:46:54 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 254-------------------- +[Apr 25, 2026 - 19:46:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0195 iter count: 1.0 +[Apr 25, 2026 - 19:46:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0195 iter count: 1.0 +[Apr 25, 2026 - 19:46:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:46:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093057 43.157616 43.004040 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.546875 135.576172 135.460938 +util 70.500000 77.000000 62.000000 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 255 : avg iteration time 172.92 seconds +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 255 : data loading time 0.00 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 255 : avg forward pass time 77.19 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 255 : backward pass time 9.13 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 255 : optimizer step time 0.34 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 255-------------------- +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 76747.0020 iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6024 iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 76747.0020iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.7070iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.7301 iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.2261 iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 152103.9688iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 31.8237iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1890108.1250iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6024iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 7.0531 iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1337468.1250iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.0695 iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 3425.5625iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.5673 iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1254.6371iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.5230 iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 8484.6406iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0212 iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:49:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 19:50:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093055 43.157608 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.631348 70.720703 70.601562 +util 96.750000 100.000000 93.000000 +[Apr 25, 2026 - 19:50:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 256 : avg iteration time 51.27 seconds +[Apr 25, 2026 - 19:50:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 256 : data loading time 0.00 +[Apr 25, 2026 - 19:50:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 256 : avg forward pass time 13.61 +[Apr 25, 2026 - 19:50:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 256 : backward pass time 11.53 +[Apr 25, 2026 - 19:50:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 256 : optimizer step time 0.99 +[Apr 25, 2026 - 19:50:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 256-------------------- +[Apr 25, 2026 - 19:50:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0212 iter count: 1.0 +[Apr 25, 2026 - 19:50:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0212 iter count: 1.0 +[Apr 25, 2026 - 19:50:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:50:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:51:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093061 43.157608 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.724121 70.738281 70.681641 +util 96.500000 99.000000 93.000000 +[Apr 25, 2026 - 19:51:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 257 : avg iteration time 46.96 seconds +[Apr 25, 2026 - 19:51:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 257 : data loading time 0.00 +[Apr 25, 2026 - 19:51:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 257 : avg forward pass time 11.45 +[Apr 25, 2026 - 19:51:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 257 : backward pass time 11.52 +[Apr 25, 2026 - 19:51:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 257 : optimizer step time 0.99 +[Apr 25, 2026 - 19:51:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 257-------------------- +[Apr 25, 2026 - 19:51:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0219 iter count: 1.0 +[Apr 25, 2026 - 19:51:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0219 iter count: 1.0 +[Apr 25, 2026 - 19:51:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:51:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:52:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093056 43.157608 43.004047 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.602051 70.720703 70.562500 +util 97.500000 99.000000 93.000000 +[Apr 25, 2026 - 19:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 258 : avg iteration time 52.91 seconds +[Apr 25, 2026 - 19:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 258 : data loading time 0.00 +[Apr 25, 2026 - 19:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 258 : avg forward pass time 14.40 +[Apr 25, 2026 - 19:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 258 : backward pass time 11.56 +[Apr 25, 2026 - 19:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 258 : optimizer step time 0.98 +[Apr 25, 2026 - 19:52:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 258-------------------- +[Apr 25, 2026 - 19:52:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0199 iter count: 1.0 +[Apr 25, 2026 - 19:52:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0199 iter count: 1.0 +[Apr 25, 2026 - 19:52:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:52:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:53:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093234 43.158329 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.650879 70.740234 70.621094 +util 97.750000 99.000000 95.000000 +[Apr 25, 2026 - 19:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 259 : avg iteration time 49.95 seconds +[Apr 25, 2026 - 19:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 259 : data loading time 0.00 +[Apr 25, 2026 - 19:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 259 : avg forward pass time 12.93 +[Apr 25, 2026 - 19:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 259 : backward pass time 11.55 +[Apr 25, 2026 - 19:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 259 : optimizer step time 0.98 +[Apr 25, 2026 - 19:53:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 259-------------------- +[Apr 25, 2026 - 19:53:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0238 iter count: 1.0 +[Apr 25, 2026 - 19:53:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0238 iter count: 1.0 +[Apr 25, 2026 - 19:53:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:53:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092972 43.157280 43.004040 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.479004 135.480469 135.478516 +util 71.750000 77.000000 68.000000 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 260 : avg iteration time 169.71 seconds +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 260 : data loading time 0.00 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 260 : avg forward pass time 75.59 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 260 : backward pass time 9.14 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 260 : optimizer step time 0.34 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 260-------------------- +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 148202.5037 iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5421 iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 148202.5037iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4811iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.9427 iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.5891 iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 277180.8438iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 6.5889 iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 3906521.0000iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5421iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.1458 iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 3990.5376iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 0.9427 iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 6.5889 iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 1.7892 iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 35.8180 iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.1750 iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 4230.3354iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0197 iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:55:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 19:56:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093037 43.157539 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.597168 70.601562 70.583984 +util 98.250000 99.000000 96.000000 +[Apr 25, 2026 - 19:56:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 261 : avg iteration time 48.38 seconds +[Apr 25, 2026 - 19:56:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 261 : data loading time 0.00 +[Apr 25, 2026 - 19:56:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 261 : avg forward pass time 12.17 +[Apr 25, 2026 - 19:56:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 261 : backward pass time 11.55 +[Apr 25, 2026 - 19:56:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 261 : optimizer step time 0.97 +[Apr 25, 2026 - 19:56:46 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 261-------------------- +[Apr 25, 2026 - 19:56:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0224 iter count: 1.0 +[Apr 25, 2026 - 19:56:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0224 iter count: 1.0 +[Apr 25, 2026 - 19:56:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:56:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:57:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093037 43.157539 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.719238 70.757812 70.603516 +util 97.000000 100.000000 94.000000 +[Apr 25, 2026 - 19:57:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 262 : avg iteration time 52.89 seconds +[Apr 25, 2026 - 19:57:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 262 : data loading time 0.00 +[Apr 25, 2026 - 19:57:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 262 : avg forward pass time 14.40 +[Apr 25, 2026 - 19:57:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 262 : backward pass time 11.57 +[Apr 25, 2026 - 19:57:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 262 : optimizer step time 0.98 +[Apr 25, 2026 - 19:57:39 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 262-------------------- +[Apr 25, 2026 - 19:57:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0214 iter count: 1.0 +[Apr 25, 2026 - 19:57:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0214 iter count: 1.0 +[Apr 25, 2026 - 19:57:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:57:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:58:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093037 43.157539 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.685059 70.718750 70.583984 +util 97.250000 99.000000 94.000000 +[Apr 25, 2026 - 19:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 263 : avg iteration time 47.02 seconds +[Apr 25, 2026 - 19:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 263 : data loading time 0.00 +[Apr 25, 2026 - 19:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 263 : avg forward pass time 11.48 +[Apr 25, 2026 - 19:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 263 : backward pass time 11.53 +[Apr 25, 2026 - 19:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 263 : optimizer step time 0.99 +[Apr 25, 2026 - 19:58:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 263-------------------- +[Apr 25, 2026 - 19:58:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0239 iter count: 1.0 +[Apr 25, 2026 - 19:58:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0239 iter count: 1.0 +[Apr 25, 2026 - 19:58:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:58:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 19:59:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093158 43.158024 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.763184 70.835938 70.544922 +util 96.000000 99.000000 92.000000 +[Apr 25, 2026 - 19:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 264 : avg iteration time 49.94 seconds +[Apr 25, 2026 - 19:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 264 : data loading time 0.00 +[Apr 25, 2026 - 19:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 264 : avg forward pass time 12.93 +[Apr 25, 2026 - 19:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 264 : backward pass time 11.55 +[Apr 25, 2026 - 19:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 264 : optimizer step time 0.98 +[Apr 25, 2026 - 19:59:16 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 264-------------------- +[Apr 25, 2026 - 19:59:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0224 iter count: 1.0 +[Apr 25, 2026 - 19:59:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0224 iter count: 1.0 +[Apr 25, 2026 - 19:59:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 19:59:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092952 43.157188 43.004051 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.531250 135.556641 135.458984 +util 62.500000 72.000000 51.000000 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 265 : avg iteration time 170.19 seconds +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 265 : data loading time 0.00 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 265 : avg forward pass time 75.75 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 265 : backward pass time 9.19 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 265 : optimizer step time 0.41 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 265-------------------- +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 159521.5293 iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5674 iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 159521.5293iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.8595iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.3070 iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.4331 iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 270736.6250iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 100.8813iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2859386.2500iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5674iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.9895 iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 159367.7344iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.8686 iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2291.9709iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.5612 iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 67673.2344iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.8114 iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 15104.3174iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0205 iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:02:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 20:02:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093132 43.157921 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.714355 70.757812 70.583984 +util 97.250000 99.000000 94.000000 +[Apr 25, 2026 - 20:02:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 266 : avg iteration time 51.47 seconds +[Apr 25, 2026 - 20:02:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 266 : data loading time 0.00 +[Apr 25, 2026 - 20:02:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 266 : avg forward pass time 13.69 +[Apr 25, 2026 - 20:02:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 266 : backward pass time 11.56 +[Apr 25, 2026 - 20:02:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 266 : optimizer step time 0.98 +[Apr 25, 2026 - 20:02:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 266-------------------- +[Apr 25, 2026 - 20:02:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0215 iter count: 1.0 +[Apr 25, 2026 - 20:02:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0215 iter count: 1.0 +[Apr 25, 2026 - 20:02:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:02:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:03:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093132 43.157921 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.645996 70.660156 70.603516 +util 98.250000 100.000000 95.000000 +[Apr 25, 2026 - 20:03:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 267 : avg iteration time 49.89 seconds +[Apr 25, 2026 - 20:03:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 267 : data loading time 0.00 +[Apr 25, 2026 - 20:03:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 267 : avg forward pass time 12.92 +[Apr 25, 2026 - 20:03:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 267 : backward pass time 11.53 +[Apr 25, 2026 - 20:03:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 267 : optimizer step time 0.98 +[Apr 25, 2026 - 20:03:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 267-------------------- +[Apr 25, 2026 - 20:03:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0220 iter count: 1.0 +[Apr 25, 2026 - 20:03:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0220 iter count: 1.0 +[Apr 25, 2026 - 20:03:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:03:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:04:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093139 43.157921 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.616699 70.640625 70.544922 +util 95.750000 99.000000 91.000000 +[Apr 25, 2026 - 20:04:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 268 : avg iteration time 52.85 seconds +[Apr 25, 2026 - 20:04:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 268 : data loading time 0.00 +[Apr 25, 2026 - 20:04:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 268 : avg forward pass time 14.38 +[Apr 25, 2026 - 20:04:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 268 : backward pass time 11.52 +[Apr 25, 2026 - 20:04:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 268 : optimizer step time 1.00 +[Apr 25, 2026 - 20:04:40 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 268-------------------- +[Apr 25, 2026 - 20:04:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0208 iter count: 1.0 +[Apr 25, 2026 - 20:04:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0208 iter count: 1.0 +[Apr 25, 2026 - 20:04:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:04:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:05:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093132 43.157921 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.611816 70.640625 70.525391 +util 98.250000 100.000000 95.000000 +[Apr 25, 2026 - 20:05:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 269 : avg iteration time 46.94 seconds +[Apr 25, 2026 - 20:05:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 269 : data loading time 0.00 +[Apr 25, 2026 - 20:05:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 269 : avg forward pass time 11.45 +[Apr 25, 2026 - 20:05:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 269 : backward pass time 11.54 +[Apr 25, 2026 - 20:05:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 269 : optimizer step time 0.97 +[Apr 25, 2026 - 20:05:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 269-------------------- +[Apr 25, 2026 - 20:05:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0236 iter count: 1.0 +[Apr 25, 2026 - 20:05:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0236 iter count: 1.0 +[Apr 25, 2026 - 20:05:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:05:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093197 43.158169 43.004040 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.570312 135.617188 135.554688 +util 70.250000 76.000000 66.000000 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 270 : avg iteration time 170.13 seconds +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 270 : data loading time 0.00 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 270 : avg forward pass time 75.80 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 270 : backward pass time 9.14 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 270 : optimizer step time 0.34 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 270-------------------- +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 134545.8945 iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5783 iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 134545.8945iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.0068iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6441 iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.9472 iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 357542.9688iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.6262 iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 7995040.5000iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5783iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.5391 iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1185.8586iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.7331 iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 95434.7422iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.7541 iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 246.7066iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.1243 iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 28240.6035iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0248 iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:08:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 20:09:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093201 43.158165 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.831543 70.855469 70.759766 +util 97.250000 99.000000 94.000000 +[Apr 25, 2026 - 20:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 271 : avg iteration time 54.18 seconds +[Apr 25, 2026 - 20:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 271 : data loading time 0.00 +[Apr 25, 2026 - 20:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 271 : avg forward pass time 15.07 +[Apr 25, 2026 - 20:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 271 : backward pass time 11.53 +[Apr 25, 2026 - 20:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 271 : optimizer step time 1.00 +[Apr 25, 2026 - 20:09:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 271-------------------- +[Apr 25, 2026 - 20:09:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0220 iter count: 1.0 +[Apr 25, 2026 - 20:09:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0220 iter count: 1.0 +[Apr 25, 2026 - 20:09:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:09:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:09:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093197 43.158165 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.738770 70.816406 70.505859 +util 97.000000 100.000000 91.000000 +[Apr 25, 2026 - 20:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 272 : avg iteration time 47.12 seconds +[Apr 25, 2026 - 20:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 272 : data loading time 0.00 +[Apr 25, 2026 - 20:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 272 : avg forward pass time 11.52 +[Apr 25, 2026 - 20:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 272 : backward pass time 11.54 +[Apr 25, 2026 - 20:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 272 : optimizer step time 0.99 +[Apr 25, 2026 - 20:09:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 272-------------------- +[Apr 25, 2026 - 20:09:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0254 iter count: 1.0 +[Apr 25, 2026 - 20:09:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0254 iter count: 1.0 +[Apr 25, 2026 - 20:09:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:09:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:10:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093196 43.158169 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.680176 70.699219 70.623047 +util 96.000000 99.000000 92.000000 +[Apr 25, 2026 - 20:10:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 273 : avg iteration time 52.83 seconds +[Apr 25, 2026 - 20:10:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 273 : data loading time 0.00 +[Apr 25, 2026 - 20:10:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 273 : avg forward pass time 14.39 +[Apr 25, 2026 - 20:10:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 273 : backward pass time 11.53 +[Apr 25, 2026 - 20:10:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 273 : optimizer step time 0.99 +[Apr 25, 2026 - 20:10:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 273-------------------- +[Apr 25, 2026 - 20:10:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0201 iter count: 1.0 +[Apr 25, 2026 - 20:10:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0201 iter count: 1.0 +[Apr 25, 2026 - 20:10:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:10:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:11:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093265 43.158417 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.587402 70.621094 70.486328 +util 95.000000 99.000000 91.000000 +[Apr 25, 2026 - 20:11:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 274 : avg iteration time 47.02 seconds +[Apr 25, 2026 - 20:11:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 274 : data loading time 0.00 +[Apr 25, 2026 - 20:11:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 274 : avg forward pass time 11.48 +[Apr 25, 2026 - 20:11:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 274 : backward pass time 11.54 +[Apr 25, 2026 - 20:11:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 274 : optimizer step time 0.99 +[Apr 25, 2026 - 20:11:39 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 274-------------------- +[Apr 25, 2026 - 20:11:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0225 iter count: 1.0 +[Apr 25, 2026 - 20:11:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0225 iter count: 1.0 +[Apr 25, 2026 - 20:11:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:11:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093283 43.158447 43.004055 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.546875 135.576172 135.537109 +util 67.000000 76.000000 61.000000 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 275 : avg iteration time 172.56 seconds +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 275 : data loading time 0.00 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 275 : avg forward pass time 77.01 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 275 : backward pass time 9.16 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 275 : optimizer step time 0.34 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 275-------------------- +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 7479819.7371 iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6248 iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 7479819.7371iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.6221iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.8101 iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 9.6886 iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 8354172.5000iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 5.0537 iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 260260448.0000iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6248iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.8667 iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 2283.2703iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.9663 iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 20592.8105iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.7300 iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 94853.6719iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.0253 iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 171189.7031iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0237 iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:14:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 20:15:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093274 43.158447 43.004047 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.592285 70.601562 70.564453 +util 96.000000 99.000000 92.000000 +[Apr 25, 2026 - 20:15:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 276 : avg iteration time 54.22 seconds +[Apr 25, 2026 - 20:15:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 276 : data loading time 0.00 +[Apr 25, 2026 - 20:15:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 276 : avg forward pass time 15.09 +[Apr 25, 2026 - 20:15:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 276 : backward pass time 11.52 +[Apr 25, 2026 - 20:15:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 276 : optimizer step time 0.97 +[Apr 25, 2026 - 20:15:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 276-------------------- +[Apr 25, 2026 - 20:15:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0223 iter count: 1.0 +[Apr 25, 2026 - 20:15:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0223 iter count: 1.0 +[Apr 25, 2026 - 20:15:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:15:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:16:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093269 43.158447 43.004040 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.499512 70.603516 70.464844 +util 97.750000 100.000000 93.000000 +[Apr 25, 2026 - 20:16:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 277 : avg iteration time 52.91 seconds +[Apr 25, 2026 - 20:16:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 277 : data loading time 0.00 +[Apr 25, 2026 - 20:16:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 277 : avg forward pass time 14.43 +[Apr 25, 2026 - 20:16:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 277 : backward pass time 11.53 +[Apr 25, 2026 - 20:16:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 277 : optimizer step time 1.00 +[Apr 25, 2026 - 20:16:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 277-------------------- +[Apr 25, 2026 - 20:16:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0222 iter count: 1.0 +[Apr 25, 2026 - 20:16:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0222 iter count: 1.0 +[Apr 25, 2026 - 20:16:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:16:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:17:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093210 43.158180 43.004047 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.641113 70.660156 70.583984 +util 96.250000 100.000000 92.000000 +[Apr 25, 2026 - 20:17:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 278 : avg iteration time 52.89 seconds +[Apr 25, 2026 - 20:17:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 278 : data loading time 0.00 +[Apr 25, 2026 - 20:17:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 278 : avg forward pass time 14.40 +[Apr 25, 2026 - 20:17:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 278 : backward pass time 11.56 +[Apr 25, 2026 - 20:17:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 278 : optimizer step time 0.98 +[Apr 25, 2026 - 20:17:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 278-------------------- +[Apr 25, 2026 - 20:17:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0230 iter count: 1.0 +[Apr 25, 2026 - 20:17:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0230 iter count: 1.0 +[Apr 25, 2026 - 20:17:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:17:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:17:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093213 43.158180 43.004059 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.528809 70.544922 70.523438 +util 95.000000 99.000000 91.000000 +[Apr 25, 2026 - 20:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 279 : avg iteration time 47.02 seconds +[Apr 25, 2026 - 20:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 279 : data loading time 0.00 +[Apr 25, 2026 - 20:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 279 : avg forward pass time 11.48 +[Apr 25, 2026 - 20:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 279 : backward pass time 11.54 +[Apr 25, 2026 - 20:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 279 : optimizer step time 0.98 +[Apr 25, 2026 - 20:17:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 279-------------------- +[Apr 25, 2026 - 20:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0243 iter count: 1.0 +[Apr 25, 2026 - 20:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0243 iter count: 1.0 +[Apr 25, 2026 - 20:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:17:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093075 43.157005 43.004265 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.498047 135.500000 135.496094 +util 67.750000 74.000000 63.000000 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 280 : avg iteration time 175.81 seconds +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 280 : data loading time 0.00 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 280 : avg forward pass time 78.63 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 280 : backward pass time 9.15 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 280 : optimizer step time 0.34 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 280-------------------- +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 25006.6309 iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6025 iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 25006.6309iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3796iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.2133 iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.1245 iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 39750.0312iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 83.6396iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 208777.2969iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6025iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.7753 iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 103837.5469iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.8428 iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 118858.3828iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 6.0428 iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 177309.0469iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.0587 iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 453.7282iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0237 iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:20:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 20:21:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093077 43.157013 43.004265 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.597168 70.621094 70.525391 +util 94.500000 99.000000 91.000000 +[Apr 25, 2026 - 20:21:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 281 : avg iteration time 48.37 seconds +[Apr 25, 2026 - 20:21:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 281 : data loading time 0.00 +[Apr 25, 2026 - 20:21:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 281 : avg forward pass time 12.17 +[Apr 25, 2026 - 20:21:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 281 : backward pass time 11.52 +[Apr 25, 2026 - 20:21:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 281 : optimizer step time 0.99 +[Apr 25, 2026 - 20:21:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 281-------------------- +[Apr 25, 2026 - 20:21:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0265 iter count: 1.0 +[Apr 25, 2026 - 20:21:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0265 iter count: 1.0 +[Apr 25, 2026 - 20:21:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:21:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:22:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093338 43.158012 43.004288 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.670410 70.679688 70.642578 +util 98.000000 99.000000 95.000000 +[Apr 25, 2026 - 20:22:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 282 : avg iteration time 52.92 seconds +[Apr 25, 2026 - 20:22:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 282 : data loading time 0.00 +[Apr 25, 2026 - 20:22:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 282 : avg forward pass time 14.44 +[Apr 25, 2026 - 20:22:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 282 : backward pass time 11.51 +[Apr 25, 2026 - 20:22:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 282 : optimizer step time 0.99 +[Apr 25, 2026 - 20:22:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 282-------------------- +[Apr 25, 2026 - 20:22:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0229 iter count: 1.0 +[Apr 25, 2026 - 20:22:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0229 iter count: 1.0 +[Apr 25, 2026 - 20:22:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:22:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:23:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093336 43.158016 43.004265 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.626465 70.660156 70.525391 +util 97.250000 100.000000 92.000000 +[Apr 25, 2026 - 20:23:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 283 : avg iteration time 52.84 seconds +[Apr 25, 2026 - 20:23:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 283 : data loading time 0.00 +[Apr 25, 2026 - 20:23:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 283 : avg forward pass time 14.40 +[Apr 25, 2026 - 20:23:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 283 : backward pass time 11.53 +[Apr 25, 2026 - 20:23:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 283 : optimizer step time 0.97 +[Apr 25, 2026 - 20:23:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 283-------------------- +[Apr 25, 2026 - 20:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0248 iter count: 1.0 +[Apr 25, 2026 - 20:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0248 iter count: 1.0 +[Apr 25, 2026 - 20:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:23:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:24:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093331 43.158028 43.004265 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.650879 70.679688 70.564453 +util 96.750000 99.000000 91.000000 +[Apr 25, 2026 - 20:24:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 284 : avg iteration time 49.80 seconds +[Apr 25, 2026 - 20:24:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 284 : data loading time 0.00 +[Apr 25, 2026 - 20:24:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 284 : avg forward pass time 12.88 +[Apr 25, 2026 - 20:24:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 284 : backward pass time 11.53 +[Apr 25, 2026 - 20:24:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 284 : optimizer step time 0.98 +[Apr 25, 2026 - 20:24:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 284-------------------- +[Apr 25, 2026 - 20:24:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0269 iter count: 1.0 +[Apr 25, 2026 - 20:24:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0269 iter count: 1.0 +[Apr 25, 2026 - 20:24:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:24:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.092975 43.156582 43.004288 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.508301 135.537109 135.421875 +util 70.000000 76.000000 65.000000 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 285 : avg iteration time 172.90 seconds +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 285 : data loading time 0.00 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 285 : avg forward pass time 77.18 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 285 : backward pass time 9.16 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 285 : optimizer step time 0.34 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 285-------------------- +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 133418.8672 iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6254 iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 133418.8672iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.6616iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.9580 iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.5807 iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 185580.2656iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 370.9096iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 3841748.0000iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6254iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.4949 iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 59267.5195iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.7215 iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 93247.3047iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.8169 iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 2067.0442iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.7063 iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 12242.8203iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0278 iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:27:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 20:27:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093108 43.157082 43.004269 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.689941 70.738281 70.544922 +util 97.250000 99.000000 92.000000 +[Apr 25, 2026 - 20:27:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 286 : avg iteration time 48.44 seconds +[Apr 25, 2026 - 20:27:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 286 : data loading time 0.00 +[Apr 25, 2026 - 20:27:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 286 : avg forward pass time 12.18 +[Apr 25, 2026 - 20:27:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 286 : backward pass time 11.55 +[Apr 25, 2026 - 20:27:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 286 : optimizer step time 0.98 +[Apr 25, 2026 - 20:27:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 286-------------------- +[Apr 25, 2026 - 20:27:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0317 iter count: 1.0 +[Apr 25, 2026 - 20:27:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0317 iter count: 1.0 +[Apr 25, 2026 - 20:27:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:27:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:28:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093476 43.158581 43.004269 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.694824 70.796875 70.388672 +util 97.750000 100.000000 93.000000 +[Apr 25, 2026 - 20:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 287 : avg iteration time 52.81 seconds +[Apr 25, 2026 - 20:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 287 : data loading time 0.00 +[Apr 25, 2026 - 20:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 287 : avg forward pass time 14.39 +[Apr 25, 2026 - 20:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 287 : backward pass time 11.53 +[Apr 25, 2026 - 20:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 287 : optimizer step time 0.97 +[Apr 25, 2026 - 20:28:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 287-------------------- +[Apr 25, 2026 - 20:28:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0278 iter count: 1.0 +[Apr 25, 2026 - 20:28:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0278 iter count: 1.0 +[Apr 25, 2026 - 20:28:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:28:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:29:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093470 43.158581 43.004269 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.763184 70.835938 70.544922 +util 94.750000 99.000000 91.000000 +[Apr 25, 2026 - 20:29:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 288 : avg iteration time 52.90 seconds +[Apr 25, 2026 - 20:29:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 288 : data loading time 0.00 +[Apr 25, 2026 - 20:29:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 288 : avg forward pass time 14.41 +[Apr 25, 2026 - 20:29:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 288 : backward pass time 11.54 +[Apr 25, 2026 - 20:29:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 288 : optimizer step time 0.98 +[Apr 25, 2026 - 20:29:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 288-------------------- +[Apr 25, 2026 - 20:29:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0287 iter count: 1.0 +[Apr 25, 2026 - 20:29:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0287 iter count: 1.0 +[Apr 25, 2026 - 20:29:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:29:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:30:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093470 43.158581 43.004269 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.768066 70.855469 70.505859 +util 98.500000 99.000000 97.000000 +[Apr 25, 2026 - 20:30:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 289 : avg iteration time 52.86 seconds +[Apr 25, 2026 - 20:30:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 289 : data loading time 0.00 +[Apr 25, 2026 - 20:30:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 289 : avg forward pass time 14.40 +[Apr 25, 2026 - 20:30:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 289 : backward pass time 11.53 +[Apr 25, 2026 - 20:30:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 289 : optimizer step time 0.99 +[Apr 25, 2026 - 20:30:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 289-------------------- +[Apr 25, 2026 - 20:30:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0255 iter count: 1.0 +[Apr 25, 2026 - 20:30:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0255 iter count: 1.0 +[Apr 25, 2026 - 20:30:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:30:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093145 43.157269 43.004284 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.483887 135.500000 135.478516 +util 67.750000 74.000000 63.000000 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 290 : avg iteration time 175.56 seconds +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 290 : data loading time 0.00 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 290 : avg forward pass time 78.52 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 290 : backward pass time 9.13 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 290 : optimizer step time 0.34 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 290-------------------- +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 60756.4473 iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4837 iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 60756.4473iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.7273iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.0728 iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.4033 iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 164672.8594iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 8.5471 iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2693933.2500iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4837iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.9947 iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 21794.1113iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 1.0728 iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 8.5471 iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.5900 iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 9701.8584iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.6440 iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 79850.5703iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0284 iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:33:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 20:34:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093210 43.157520 43.004292 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.645996 70.662109 70.640625 +util 95.000000 100.000000 91.000000 +[Apr 25, 2026 - 20:34:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 291 : avg iteration time 48.38 seconds +[Apr 25, 2026 - 20:34:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 291 : data loading time 0.00 +[Apr 25, 2026 - 20:34:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 291 : avg forward pass time 12.16 +[Apr 25, 2026 - 20:34:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 291 : backward pass time 11.55 +[Apr 25, 2026 - 20:34:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 291 : optimizer step time 0.98 +[Apr 25, 2026 - 20:34:22 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 291-------------------- +[Apr 25, 2026 - 20:34:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0310 iter count: 1.0 +[Apr 25, 2026 - 20:34:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0310 iter count: 1.0 +[Apr 25, 2026 - 20:34:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:34:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:35:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093283 43.157764 43.004314 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.621582 70.679688 70.447266 +util 98.500000 99.000000 97.000000 +[Apr 25, 2026 - 20:35:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 292 : avg iteration time 52.86 seconds +[Apr 25, 2026 - 20:35:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 292 : data loading time 0.00 +[Apr 25, 2026 - 20:35:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 292 : avg forward pass time 14.38 +[Apr 25, 2026 - 20:35:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 292 : backward pass time 11.54 +[Apr 25, 2026 - 20:35:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 292 : optimizer step time 0.99 +[Apr 25, 2026 - 20:35:15 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 292-------------------- +[Apr 25, 2026 - 20:35:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0273 iter count: 1.0 +[Apr 25, 2026 - 20:35:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0273 iter count: 1.0 +[Apr 25, 2026 - 20:35:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:35:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:36:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093266 43.157764 43.004269 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.460449 70.464844 70.447266 +util 98.250000 99.000000 96.000000 +[Apr 25, 2026 - 20:36:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 293 : avg iteration time 49.88 seconds +[Apr 25, 2026 - 20:36:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 293 : data loading time 0.00 +[Apr 25, 2026 - 20:36:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 293 : avg forward pass time 12.90 +[Apr 25, 2026 - 20:36:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 293 : backward pass time 11.54 +[Apr 25, 2026 - 20:36:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 293 : optimizer step time 0.98 +[Apr 25, 2026 - 20:36:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 293-------------------- +[Apr 25, 2026 - 20:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0266 iter count: 1.0 +[Apr 25, 2026 - 20:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0266 iter count: 1.0 +[Apr 25, 2026 - 20:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:36:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:36:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093459 43.158516 43.004292 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.470215 70.484375 70.427734 +util 97.000000 99.000000 94.000000 +[Apr 25, 2026 - 20:36:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 294 : avg iteration time 49.89 seconds +[Apr 25, 2026 - 20:36:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 294 : data loading time 0.00 +[Apr 25, 2026 - 20:36:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 294 : avg forward pass time 12.92 +[Apr 25, 2026 - 20:36:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 294 : backward pass time 11.52 +[Apr 25, 2026 - 20:36:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 294 : optimizer step time 0.99 +[Apr 25, 2026 - 20:36:54 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 294-------------------- +[Apr 25, 2026 - 20:36:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0234 iter count: 1.0 +[Apr 25, 2026 - 20:36:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0234 iter count: 1.0 +[Apr 25, 2026 - 20:36:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:36:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093238 43.157654 43.004269 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.458984 135.458984 135.458984 +util 70.250000 74.000000 64.000000 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 295 : avg iteration time 175.87 seconds +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 295 : data loading time 0.00 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 295 : avg forward pass time 78.65 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 295 : backward pass time 9.16 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 295 : optimizer step time 0.34 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 295-------------------- +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 74427.1560 iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5117 iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 74427.1560iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3195iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.0294 iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.6481 iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 185237.4375iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 7.8358 iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 4395966.0000iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5117iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.3848 iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 6435.2168iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.3918 iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 6526.8516iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.6607 iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 204.6824iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.9770 iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 21036.6484iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0283 iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:39:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 20:40:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093243 43.157654 43.004269 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.606934 70.623047 70.601562 +util 98.500000 99.000000 97.000000 +[Apr 25, 2026 - 20:40:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 296 : avg iteration time 48.44 seconds +[Apr 25, 2026 - 20:40:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 296 : data loading time 0.00 +[Apr 25, 2026 - 20:40:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 296 : avg forward pass time 12.18 +[Apr 25, 2026 - 20:40:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 296 : backward pass time 11.55 +[Apr 25, 2026 - 20:40:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 296 : optimizer step time 0.99 +[Apr 25, 2026 - 20:40:39 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 296-------------------- +[Apr 25, 2026 - 20:40:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0273 iter count: 1.0 +[Apr 25, 2026 - 20:40:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0273 iter count: 1.0 +[Apr 25, 2026 - 20:40:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:40:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:41:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093244 43.157654 43.004292 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.499512 70.603516 70.464844 +util 98.250000 99.000000 96.000000 +[Apr 25, 2026 - 20:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 297 : avg iteration time 46.99 seconds +[Apr 25, 2026 - 20:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 297 : data loading time 0.00 +[Apr 25, 2026 - 20:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 297 : avg forward pass time 11.46 +[Apr 25, 2026 - 20:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 297 : backward pass time 11.54 +[Apr 25, 2026 - 20:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 297 : optimizer step time 0.98 +[Apr 25, 2026 - 20:41:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 297-------------------- +[Apr 25, 2026 - 20:41:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0293 iter count: 1.0 +[Apr 25, 2026 - 20:41:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0293 iter count: 1.0 +[Apr 25, 2026 - 20:41:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:41:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:42:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093363 43.158138 43.004284 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.645996 70.662109 70.640625 +util 96.000000 99.000000 92.000000 +[Apr 25, 2026 - 20:42:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 298 : avg iteration time 52.85 seconds +[Apr 25, 2026 - 20:42:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 298 : data loading time 0.00 +[Apr 25, 2026 - 20:42:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 298 : avg forward pass time 14.39 +[Apr 25, 2026 - 20:42:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 298 : backward pass time 11.52 +[Apr 25, 2026 - 20:42:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 298 : optimizer step time 1.00 +[Apr 25, 2026 - 20:42:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 298-------------------- +[Apr 25, 2026 - 20:42:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0256 iter count: 1.0 +[Apr 25, 2026 - 20:42:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0256 iter count: 1.0 +[Apr 25, 2026 - 20:42:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:42:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:43:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093423 43.158382 43.004269 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.675293 70.699219 70.603516 +util 98.250000 99.000000 96.000000 +[Apr 25, 2026 - 20:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 299 : avg iteration time 49.87 seconds +[Apr 25, 2026 - 20:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 299 : data loading time 0.00 +[Apr 25, 2026 - 20:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 299 : avg forward pass time 12.90 +[Apr 25, 2026 - 20:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 299 : backward pass time 11.54 +[Apr 25, 2026 - 20:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 299 : optimizer step time 0.99 +[Apr 25, 2026 - 20:43:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 299-------------------- +[Apr 25, 2026 - 20:43:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0282 iter count: 1.0 +[Apr 25, 2026 - 20:43:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0282 iter count: 1.0 +[Apr 25, 2026 - 20:43:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:43:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093426 43.158405 43.004269 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.483887 135.498047 135.441406 +util 71.250000 77.000000 67.000000 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 300 : avg iteration time 173.01 seconds +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 300 : data loading time 0.00 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 300 : avg forward pass time 77.25 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 300 : backward pass time 9.12 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 300 : optimizer step time 0.34 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 300-------------------- +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 70000.2798 iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5262 iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 70000.2798iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.7074iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.9880 iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.9175 iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 132599.5156iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 7.2136 iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1019776.8750iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5262iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.9175 iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1019776.8750iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.6207 iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 563254.8750iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.1086 iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 27372.0977iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.8890 iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 963174.0000iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0282 iter count: 1.0 +[Apr 25, 2026 - 20:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 25, 2026 - 20:46:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 15.62 seconds +[Apr 25, 2026 - 20:46:17 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 300 +[Apr 25, 2026 - 20:46:17 | INFO | fastgen.utils.checkpointer:save:301 ] Saving FSDP model to prefix /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000300 +[Apr 25, 2026 - 20:46:31 | SUCCESS | fastgen.utils.checkpointer:save:329 ] Model saved at iteration 300 +[Apr 25, 2026 - 20:46:32 | INFO | fastgen.trainer:validate:358 ] Validation iteration 300 +[Apr 25, 2026 - 20:46:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 20:46:34 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 2.2s +[Apr 25, 2026 - 20:46:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 20:46:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 20:46:43 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 20:46:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 20:46:48 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 20:46:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 20:46:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 20:46:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 20:46:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 20:47:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 20:47:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 20:47:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 20:47:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 20:47:11 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 20:47:14 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 20:47:16 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 20:47:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 20:47:21 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 20:47:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging val0 stats at iteration 300-------------------- +[Apr 25, 2026 - 20:47:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 25, 2026 - 20:47:41 | INFO | fastgen.callbacks.wandb:on_validation_end:683 ] Logged 10 val videos at iteration 300 +[Apr 25, 2026 - 20:48:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094365 43.162186 43.004211 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.763184 71.130859 70.640625 +util 95.250000 99.000000 91.000000 +[Apr 25, 2026 - 20:48:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 301 : avg iteration time 148.90 seconds +[Apr 25, 2026 - 20:48:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 301 : data loading time 0.00 +[Apr 25, 2026 - 20:48:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 301 : avg forward pass time 12.54 +[Apr 25, 2026 - 20:48:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 301 : backward pass time 11.55 +[Apr 25, 2026 - 20:48:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 301 : optimizer step time 0.98 +[Apr 25, 2026 - 20:48:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 301-------------------- +[Apr 25, 2026 - 20:48:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0242 iter count: 1.0 +[Apr 25, 2026 - 20:48:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0242 iter count: 1.0 +[Apr 25, 2026 - 20:48:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:48:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:49:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094366 43.162186 43.004211 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.782715 71.091797 70.679688 +util 95.250000 99.000000 91.000000 +[Apr 25, 2026 - 20:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 302 : avg iteration time 50.01 seconds +[Apr 25, 2026 - 20:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 302 : data loading time 0.00 +[Apr 25, 2026 - 20:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 302 : avg forward pass time 12.93 +[Apr 25, 2026 - 20:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 302 : backward pass time 11.60 +[Apr 25, 2026 - 20:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 302 : optimizer step time 0.98 +[Apr 25, 2026 - 20:49:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 302-------------------- +[Apr 25, 2026 - 20:49:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0251 iter count: 1.0 +[Apr 25, 2026 - 20:49:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0251 iter count: 1.0 +[Apr 25, 2026 - 20:49:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:49:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:50:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094367 43.162186 43.004211 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.748535 70.955078 70.679688 +util 97.250000 99.000000 94.000000 +[Apr 25, 2026 - 20:50:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 303 : avg iteration time 50.08 seconds +[Apr 25, 2026 - 20:50:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 303 : data loading time 0.00 +[Apr 25, 2026 - 20:50:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 303 : avg forward pass time 12.96 +[Apr 25, 2026 - 20:50:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 303 : backward pass time 11.58 +[Apr 25, 2026 - 20:50:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 303 : optimizer step time 0.97 +[Apr 25, 2026 - 20:50:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 303-------------------- +[Apr 25, 2026 - 20:50:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0261 iter count: 1.0 +[Apr 25, 2026 - 20:50:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0261 iter count: 1.0 +[Apr 25, 2026 - 20:50:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:50:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:51:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094431 43.162437 43.004211 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.812012 71.150391 70.699219 +util 98.000000 100.000000 94.000000 +[Apr 25, 2026 - 20:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 304 : avg iteration time 52.87 seconds +[Apr 25, 2026 - 20:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 304 : data loading time 0.00 +[Apr 25, 2026 - 20:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 304 : avg forward pass time 14.40 +[Apr 25, 2026 - 20:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 304 : backward pass time 11.51 +[Apr 25, 2026 - 20:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 304 : optimizer step time 0.99 +[Apr 25, 2026 - 20:51:03 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 304-------------------- +[Apr 25, 2026 - 20:51:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0251 iter count: 1.0 +[Apr 25, 2026 - 20:51:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0251 iter count: 1.0 +[Apr 25, 2026 - 20:51:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:51:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093675 43.158043 43.004662 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.532227 135.556641 135.460938 +util 73.750000 78.000000 67.000000 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 305 : avg iteration time 178.84 seconds +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 305 : data loading time 0.00 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 305 : avg forward pass time 80.15 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 305 : backward pass time 9.15 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 305 : optimizer step time 0.34 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 305-------------------- +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 207259.0625 iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5740 iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 207259.0625iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5574iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.0877 iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.7053 iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 351275.4688iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 8.8054 iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 4928876.5000iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5740iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.9868 iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 158503.4531iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.3732 iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 46462.9414iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.8721 iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 126030.7031iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.6651 iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1525.6057iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0236 iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:54:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 20:54:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093679 43.158043 43.004662 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.753418 70.757812 70.740234 +util 98.250000 99.000000 96.000000 +[Apr 25, 2026 - 20:54:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 306 : avg iteration time 48.51 seconds +[Apr 25, 2026 - 20:54:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 306 : data loading time 0.00 +[Apr 25, 2026 - 20:54:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 306 : avg forward pass time 12.20 +[Apr 25, 2026 - 20:54:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 306 : backward pass time 11.56 +[Apr 25, 2026 - 20:54:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 306 : optimizer step time 0.99 +[Apr 25, 2026 - 20:54:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 306-------------------- +[Apr 25, 2026 - 20:54:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0259 iter count: 1.0 +[Apr 25, 2026 - 20:54:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0259 iter count: 1.0 +[Apr 25, 2026 - 20:54:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:54:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:55:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093982 43.159294 43.004662 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.802246 70.835938 70.701172 +util 94.750000 99.000000 90.000000 +[Apr 25, 2026 - 20:55:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 307 : avg iteration time 49.90 seconds +[Apr 25, 2026 - 20:55:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 307 : data loading time 0.00 +[Apr 25, 2026 - 20:55:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 307 : avg forward pass time 12.92 +[Apr 25, 2026 - 20:55:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 307 : backward pass time 11.55 +[Apr 25, 2026 - 20:55:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 307 : optimizer step time 0.97 +[Apr 25, 2026 - 20:55:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 307-------------------- +[Apr 25, 2026 - 20:55:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0275 iter count: 1.0 +[Apr 25, 2026 - 20:55:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0275 iter count: 1.0 +[Apr 25, 2026 - 20:55:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:55:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:56:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094049 43.159542 43.004662 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.772949 70.835938 70.583984 +util 97.750000 99.000000 95.000000 +[Apr 25, 2026 - 20:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 308 : avg iteration time 49.89 seconds +[Apr 25, 2026 - 20:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 308 : data loading time 0.00 +[Apr 25, 2026 - 20:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 308 : avg forward pass time 12.91 +[Apr 25, 2026 - 20:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 308 : backward pass time 11.54 +[Apr 25, 2026 - 20:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 308 : optimizer step time 0.98 +[Apr 25, 2026 - 20:56:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 308-------------------- +[Apr 25, 2026 - 20:56:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0259 iter count: 1.0 +[Apr 25, 2026 - 20:56:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0259 iter count: 1.0 +[Apr 25, 2026 - 20:56:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:56:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 20:57:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094049 43.159542 43.004662 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.714355 70.757812 70.583984 +util 97.750000 100.000000 93.000000 +[Apr 25, 2026 - 20:57:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 309 : avg iteration time 49.85 seconds +[Apr 25, 2026 - 20:57:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 309 : data loading time 0.00 +[Apr 25, 2026 - 20:57:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 309 : avg forward pass time 12.90 +[Apr 25, 2026 - 20:57:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 309 : backward pass time 11.54 +[Apr 25, 2026 - 20:57:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 309 : optimizer step time 0.98 +[Apr 25, 2026 - 20:57:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 309-------------------- +[Apr 25, 2026 - 20:57:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0278 iter count: 1.0 +[Apr 25, 2026 - 20:57:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0278 iter count: 1.0 +[Apr 25, 2026 - 20:57:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 20:57:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093589 43.157700 43.004662 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.512207 135.537109 135.441406 +util 67.250000 73.000000 63.000000 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 310 : avg iteration time 169.72 seconds +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 310 : data loading time 0.00 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 310 : avg forward pass time 75.57 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 310 : backward pass time 9.16 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 310 : optimizer step time 0.34 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 310-------------------- +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 113406.1768 iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4808 iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 113406.1768iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.7889iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.1643 iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.4742 iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 179309.3281iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 75.8450iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 3104719.5000iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4808iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.6169 iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1385.4918iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.1579 iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 223178.5156iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.2235 iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 34440.3867iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.9564 iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 20189.0215iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0308 iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:00:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 21:01:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093594 43.157700 43.004662 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.587402 70.720703 70.542969 +util 95.250000 99.000000 91.000000 +[Apr 25, 2026 - 21:01:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 311 : avg iteration time 51.35 seconds +[Apr 25, 2026 - 21:01:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 311 : data loading time 0.00 +[Apr 25, 2026 - 21:01:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 311 : avg forward pass time 13.65 +[Apr 25, 2026 - 21:01:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 311 : backward pass time 11.52 +[Apr 25, 2026 - 21:01:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 311 : optimizer step time 1.00 +[Apr 25, 2026 - 21:01:01 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 311-------------------- +[Apr 25, 2026 - 21:01:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0287 iter count: 1.0 +[Apr 25, 2026 - 21:01:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0287 iter count: 1.0 +[Apr 25, 2026 - 21:01:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:01:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:01:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093893 43.158939 43.004662 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.538574 70.701172 70.484375 +util 95.750000 100.000000 92.000000 +[Apr 25, 2026 - 21:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 312 : avg iteration time 50.04 seconds +[Apr 25, 2026 - 21:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 312 : data loading time 0.00 +[Apr 25, 2026 - 21:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 312 : avg forward pass time 12.99 +[Apr 25, 2026 - 21:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 312 : backward pass time 11.52 +[Apr 25, 2026 - 21:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 312 : optimizer step time 0.99 +[Apr 25, 2026 - 21:01:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 312-------------------- +[Apr 25, 2026 - 21:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0263 iter count: 1.0 +[Apr 25, 2026 - 21:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0263 iter count: 1.0 +[Apr 25, 2026 - 21:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:01:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:02:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093897 43.158955 43.004662 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.694824 70.740234 70.679688 +util 99.000000 100.000000 98.000000 +[Apr 25, 2026 - 21:02:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 313 : avg iteration time 49.94 seconds +[Apr 25, 2026 - 21:02:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 313 : data loading time 0.00 +[Apr 25, 2026 - 21:02:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 313 : avg forward pass time 12.92 +[Apr 25, 2026 - 21:02:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 313 : backward pass time 11.58 +[Apr 25, 2026 - 21:02:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 313 : optimizer step time 0.98 +[Apr 25, 2026 - 21:02:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 313-------------------- +[Apr 25, 2026 - 21:02:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0259 iter count: 1.0 +[Apr 25, 2026 - 21:02:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0259 iter count: 1.0 +[Apr 25, 2026 - 21:02:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:02:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:03:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093900 43.158939 43.004662 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.670410 70.701172 70.660156 +util 97.750000 100.000000 92.000000 +[Apr 25, 2026 - 21:03:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 314 : avg iteration time 46.94 seconds +[Apr 25, 2026 - 21:03:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 314 : data loading time 0.00 +[Apr 25, 2026 - 21:03:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 314 : avg forward pass time 11.44 +[Apr 25, 2026 - 21:03:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 314 : backward pass time 11.54 +[Apr 25, 2026 - 21:03:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 314 : optimizer step time 0.98 +[Apr 25, 2026 - 21:03:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 314-------------------- +[Apr 25, 2026 - 21:03:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0285 iter count: 1.0 +[Apr 25, 2026 - 21:03:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0285 iter count: 1.0 +[Apr 25, 2026 - 21:03:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:03:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093727 43.158253 43.004662 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.488281 135.498047 135.458984 +util 67.500000 72.000000 60.000000 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 315 : avg iteration time 172.59 seconds +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 315 : data loading time 0.00 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 315 : avg forward pass time 77.02 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 315 : backward pass time 9.16 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 315 : optimizer step time 0.34 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 315-------------------- +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 144187.0376 iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5224 iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 144187.0376iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.9405iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.4803 iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.5230 iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 253115.3750iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 19.3094iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 3422733.2500iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5224iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.0108 iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 55.7912 iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.2784 iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 5201.8936iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.2140 iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 4573.0542iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.5099 iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 8264.5273iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0247 iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:06:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 21:07:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093725 43.158264 43.004665 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.675293 70.720703 70.660156 +util 99.000000 100.000000 97.000000 +[Apr 25, 2026 - 21:07:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 316 : avg iteration time 51.27 seconds +[Apr 25, 2026 - 21:07:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 316 : data loading time 0.00 +[Apr 25, 2026 - 21:07:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 316 : avg forward pass time 13.62 +[Apr 25, 2026 - 21:07:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 316 : backward pass time 11.52 +[Apr 25, 2026 - 21:07:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 316 : optimizer step time 0.98 +[Apr 25, 2026 - 21:07:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 316-------------------- +[Apr 25, 2026 - 21:07:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0290 iter count: 1.0 +[Apr 25, 2026 - 21:07:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0290 iter count: 1.0 +[Apr 25, 2026 - 21:07:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:07:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:07:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094042 43.159477 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.641113 70.660156 70.583984 +util 97.750000 100.000000 93.000000 +[Apr 25, 2026 - 21:07:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 317 : avg iteration time 46.95 seconds +[Apr 25, 2026 - 21:07:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 317 : data loading time 0.00 +[Apr 25, 2026 - 21:07:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 317 : avg forward pass time 11.44 +[Apr 25, 2026 - 21:07:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 317 : backward pass time 11.54 +[Apr 25, 2026 - 21:07:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 317 : optimizer step time 0.99 +[Apr 25, 2026 - 21:07:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 317-------------------- +[Apr 25, 2026 - 21:07:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0281 iter count: 1.0 +[Apr 25, 2026 - 21:07:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0281 iter count: 1.0 +[Apr 25, 2026 - 21:07:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:07:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:08:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094034 43.159500 43.004665 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.494629 70.583984 70.464844 +util 95.750000 100.000000 92.000000 +[Apr 25, 2026 - 21:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 318 : avg iteration time 49.91 seconds +[Apr 25, 2026 - 21:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 318 : data loading time 0.00 +[Apr 25, 2026 - 21:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 318 : avg forward pass time 12.93 +[Apr 25, 2026 - 21:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 318 : backward pass time 11.53 +[Apr 25, 2026 - 21:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 318 : optimizer step time 0.98 +[Apr 25, 2026 - 21:08:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 318-------------------- +[Apr 25, 2026 - 21:08:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0233 iter count: 1.0 +[Apr 25, 2026 - 21:08:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0233 iter count: 1.0 +[Apr 25, 2026 - 21:08:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:08:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:09:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094043 43.159500 43.004684 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.567871 70.583984 70.562500 +util 95.250000 100.000000 91.000000 +[Apr 25, 2026 - 21:09:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 319 : avg iteration time 47.00 seconds +[Apr 25, 2026 - 21:09:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 319 : data loading time 0.00 +[Apr 25, 2026 - 21:09:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 319 : avg forward pass time 11.48 +[Apr 25, 2026 - 21:09:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 319 : backward pass time 11.53 +[Apr 25, 2026 - 21:09:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 319 : optimizer step time 0.98 +[Apr 25, 2026 - 21:09:36 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 319-------------------- +[Apr 25, 2026 - 21:09:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0293 iter count: 1.0 +[Apr 25, 2026 - 21:09:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0293 iter count: 1.0 +[Apr 25, 2026 - 21:09:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:09:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093745 43.158344 43.004665 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.536133 135.554688 135.480469 +util 67.500000 72.000000 61.000000 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 320 : avg iteration time 175.67 seconds +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 320 : data loading time 0.00 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 320 : avg forward pass time 78.57 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 320 : backward pass time 9.11 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 320 : optimizer step time 0.34 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 320-------------------- +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 425221.8794 iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6287 iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 425221.8794iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.9467iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6358 iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.2810 iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 809803.7500iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.5666 iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 15589007.0000iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6287iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 1.5617 iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 22.7214 iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.7681 iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 13852.7314iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.5263 iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 63110.4062iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.5255 iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 465530.0312iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0277 iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:12:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 21:13:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093752 43.158344 43.004665 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.680176 70.718750 70.564453 +util 96.250000 100.000000 89.000000 +[Apr 25, 2026 - 21:13:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 321 : avg iteration time 48.48 seconds +[Apr 25, 2026 - 21:13:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 321 : data loading time 0.00 +[Apr 25, 2026 - 21:13:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 321 : avg forward pass time 12.19 +[Apr 25, 2026 - 21:13:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 321 : backward pass time 11.53 +[Apr 25, 2026 - 21:13:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 321 : optimizer step time 1.00 +[Apr 25, 2026 - 21:13:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 321-------------------- +[Apr 25, 2026 - 21:13:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0268 iter count: 1.0 +[Apr 25, 2026 - 21:13:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0268 iter count: 1.0 +[Apr 25, 2026 - 21:13:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:13:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:14:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093746 43.158344 43.004665 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.650879 70.679688 70.564453 +util 96.000000 99.000000 93.000000 +[Apr 25, 2026 - 21:14:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 322 : avg iteration time 52.83 seconds +[Apr 25, 2026 - 21:14:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 322 : data loading time 0.00 +[Apr 25, 2026 - 21:14:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 322 : avg forward pass time 14.37 +[Apr 25, 2026 - 21:14:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 322 : backward pass time 11.55 +[Apr 25, 2026 - 21:14:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 322 : optimizer step time 0.98 +[Apr 25, 2026 - 21:14:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 322-------------------- +[Apr 25, 2026 - 21:14:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0225 iter count: 1.0 +[Apr 25, 2026 - 21:14:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0225 iter count: 1.0 +[Apr 25, 2026 - 21:14:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:14:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:15:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093746 43.158344 43.004665 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.577637 70.582031 70.564453 +util 97.250000 99.000000 92.000000 +[Apr 25, 2026 - 21:15:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 323 : avg iteration time 52.91 seconds +[Apr 25, 2026 - 21:15:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 323 : data loading time 0.00 +[Apr 25, 2026 - 21:15:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 323 : avg forward pass time 14.44 +[Apr 25, 2026 - 21:15:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 323 : backward pass time 11.51 +[Apr 25, 2026 - 21:15:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 323 : optimizer step time 0.99 +[Apr 25, 2026 - 21:15:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 323-------------------- +[Apr 25, 2026 - 21:15:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0220 iter count: 1.0 +[Apr 25, 2026 - 21:15:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0220 iter count: 1.0 +[Apr 25, 2026 - 21:15:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:15:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:15:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093865 43.158821 43.004665 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.636230 70.660156 70.564453 +util 97.000000 100.000000 91.000000 +[Apr 25, 2026 - 21:15:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 324 : avg iteration time 46.99 seconds +[Apr 25, 2026 - 21:15:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 324 : data loading time 0.00 +[Apr 25, 2026 - 21:15:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 324 : avg forward pass time 11.47 +[Apr 25, 2026 - 21:15:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 324 : backward pass time 11.54 +[Apr 25, 2026 - 21:15:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 324 : optimizer step time 0.99 +[Apr 25, 2026 - 21:15:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 324-------------------- +[Apr 25, 2026 - 21:15:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0250 iter count: 1.0 +[Apr 25, 2026 - 21:15:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0250 iter count: 1.0 +[Apr 25, 2026 - 21:15:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:15:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093822 43.158649 43.004665 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.487793 135.498047 135.460938 +util 68.500000 77.000000 58.000000 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 325 : avg iteration time 170.08 seconds +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 325 : data loading time 0.00 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 325 : avg forward pass time 75.73 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 325 : backward pass time 9.17 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 325 : optimizer step time 0.35 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 325-------------------- +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 98038.9930 iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4842 iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 98038.9930iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.6063iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7455 iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.3210 iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 204170.4688iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.4413 iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2285219.2500iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4842iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.3659 iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 838.6747iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.7842 iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 14304.2734iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.1578 iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 30201.6230iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.3390 iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 43387.6797iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0270 iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:18:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 21:19:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093884 43.158894 43.004665 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.665527 70.681641 70.660156 +util 98.250000 100.000000 95.000000 +[Apr 25, 2026 - 21:19:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 326 : avg iteration time 51.37 seconds +[Apr 25, 2026 - 21:19:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 326 : data loading time 0.00 +[Apr 25, 2026 - 21:19:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 326 : avg forward pass time 13.66 +[Apr 25, 2026 - 21:19:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 326 : backward pass time 11.53 +[Apr 25, 2026 - 21:19:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 326 : optimizer step time 0.97 +[Apr 25, 2026 - 21:19:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 326-------------------- +[Apr 25, 2026 - 21:19:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0233 iter count: 1.0 +[Apr 25, 2026 - 21:19:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0233 iter count: 1.0 +[Apr 25, 2026 - 21:19:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:19:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:20:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093882 43.158886 43.004665 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.665527 70.699219 70.564453 +util 98.250000 99.000000 96.000000 +[Apr 25, 2026 - 21:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 327 : avg iteration time 52.81 seconds +[Apr 25, 2026 - 21:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 327 : data loading time 0.00 +[Apr 25, 2026 - 21:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 327 : avg forward pass time 14.38 +[Apr 25, 2026 - 21:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 327 : backward pass time 11.53 +[Apr 25, 2026 - 21:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 327 : optimizer step time 0.98 +[Apr 25, 2026 - 21:20:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 327-------------------- +[Apr 25, 2026 - 21:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0224 iter count: 1.0 +[Apr 25, 2026 - 21:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0224 iter count: 1.0 +[Apr 25, 2026 - 21:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:20:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:21:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093882 43.158886 43.004665 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.645996 70.679688 70.544922 +util 96.000000 99.000000 93.000000 +[Apr 25, 2026 - 21:21:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 328 : avg iteration time 52.88 seconds +[Apr 25, 2026 - 21:21:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 328 : data loading time 0.00 +[Apr 25, 2026 - 21:21:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 328 : avg forward pass time 14.42 +[Apr 25, 2026 - 21:21:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 328 : backward pass time 11.52 +[Apr 25, 2026 - 21:21:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 328 : optimizer step time 0.98 +[Apr 25, 2026 - 21:21:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 328-------------------- +[Apr 25, 2026 - 21:21:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0233 iter count: 1.0 +[Apr 25, 2026 - 21:21:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0233 iter count: 1.0 +[Apr 25, 2026 - 21:21:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:21:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:22:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093882 43.158886 43.004665 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.631348 70.660156 70.544922 +util 97.000000 99.000000 94.000000 +[Apr 25, 2026 - 21:22:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 329 : avg iteration time 50.01 seconds +[Apr 25, 2026 - 21:22:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 329 : data loading time 0.00 +[Apr 25, 2026 - 21:22:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 329 : avg forward pass time 12.98 +[Apr 25, 2026 - 21:22:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 329 : backward pass time 11.54 +[Apr 25, 2026 - 21:22:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 329 : optimizer step time 0.99 +[Apr 25, 2026 - 21:22:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 329-------------------- +[Apr 25, 2026 - 21:22:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0247 iter count: 1.0 +[Apr 25, 2026 - 21:22:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0247 iter count: 1.0 +[Apr 25, 2026 - 21:22:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:22:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093886 43.158901 43.004665 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.537598 135.576172 135.421875 +util 67.500000 71.000000 61.000000 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 330 : avg iteration time 178.64 seconds +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 330 : data loading time 0.00 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 330 : avg forward pass time 80.04 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 330 : backward pass time 9.15 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 330 : optimizer step time 0.34 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 330-------------------- +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 118653.4341 iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6237 iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 118653.4341iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5681iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.4183 iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.2887 iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 153407.2188iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 17.0562iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2142257.5000iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6237iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.6938 iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1615.7494iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.6468 iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 199.0648iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.5791 iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1284.4830iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.7184 iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1697.1936iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0230 iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:25:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 21:26:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093883 43.158890 43.004665 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.694824 70.699219 70.681641 +util 98.750000 100.000000 97.000000 +[Apr 25, 2026 - 21:26:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 331 : avg iteration time 51.47 seconds +[Apr 25, 2026 - 21:26:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 331 : data loading time 0.00 +[Apr 25, 2026 - 21:26:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 331 : avg forward pass time 13.72 +[Apr 25, 2026 - 21:26:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 331 : backward pass time 11.51 +[Apr 25, 2026 - 21:26:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 331 : optimizer step time 0.98 +[Apr 25, 2026 - 21:26:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 331-------------------- +[Apr 25, 2026 - 21:26:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0276 iter count: 1.0 +[Apr 25, 2026 - 21:26:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0276 iter count: 1.0 +[Apr 25, 2026 - 21:26:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:26:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:26:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093882 43.158886 43.004665 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.689941 70.738281 70.544922 +util 98.000000 100.000000 94.000000 +[Apr 25, 2026 - 21:26:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 332 : avg iteration time 52.96 seconds +[Apr 25, 2026 - 21:26:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 332 : data loading time 0.00 +[Apr 25, 2026 - 21:26:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 332 : avg forward pass time 14.43 +[Apr 25, 2026 - 21:26:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 332 : backward pass time 11.55 +[Apr 25, 2026 - 21:26:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 332 : optimizer step time 0.98 +[Apr 25, 2026 - 21:26:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 332-------------------- +[Apr 25, 2026 - 21:26:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0287 iter count: 1.0 +[Apr 25, 2026 - 21:26:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0287 iter count: 1.0 +[Apr 25, 2026 - 21:26:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:26:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:27:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093892 43.158886 43.004688 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.572754 70.582031 70.544922 +util 98.500000 100.000000 95.000000 +[Apr 25, 2026 - 21:27:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 333 : avg iteration time 47.02 seconds +[Apr 25, 2026 - 21:27:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 333 : data loading time 0.00 +[Apr 25, 2026 - 21:27:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 333 : avg forward pass time 11.47 +[Apr 25, 2026 - 21:27:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 333 : backward pass time 11.55 +[Apr 25, 2026 - 21:27:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 333 : optimizer step time 0.98 +[Apr 25, 2026 - 21:27:40 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 333-------------------- +[Apr 25, 2026 - 21:27:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0290 iter count: 1.0 +[Apr 25, 2026 - 21:27:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0290 iter count: 1.0 +[Apr 25, 2026 - 21:27:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:27:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:28:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093883 43.158886 43.004669 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.704590 70.718750 70.662109 +util 96.250000 99.000000 93.000000 +[Apr 25, 2026 - 21:28:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 334 : avg iteration time 52.86 seconds +[Apr 25, 2026 - 21:28:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 334 : data loading time 0.00 +[Apr 25, 2026 - 21:28:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 334 : avg forward pass time 14.39 +[Apr 25, 2026 - 21:28:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 334 : backward pass time 11.55 +[Apr 25, 2026 - 21:28:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 334 : optimizer step time 0.98 +[Apr 25, 2026 - 21:28:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 334-------------------- +[Apr 25, 2026 - 21:28:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0276 iter count: 1.0 +[Apr 25, 2026 - 21:28:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0276 iter count: 1.0 +[Apr 25, 2026 - 21:28:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:28:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093555 43.157566 43.004673 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.498047 135.498047 135.498047 +util 67.000000 73.000000 63.000000 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 335 : avg iteration time 175.83 seconds +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 335 : data loading time 0.00 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 335 : avg forward pass time 78.64 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 335 : backward pass time 9.14 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 335 : optimizer step time 0.34 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 335-------------------- +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 379950.6387 iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6473 iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 379950.6387iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4604iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6364 iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.1349 iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 446357.1250iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.5707 iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 11638718.0000iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6473iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.0923 iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 3585.5994iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.9763 iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2842.8823iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.3529 iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 6038.3550iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.5447 iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 8860.2539iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0277 iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:31:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 21:32:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093554 43.157566 43.004673 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.606934 70.640625 70.505859 +util 95.250000 100.000000 91.000000 +[Apr 25, 2026 - 21:32:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 336 : avg iteration time 51.39 seconds +[Apr 25, 2026 - 21:32:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 336 : data loading time 0.00 +[Apr 25, 2026 - 21:32:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 336 : avg forward pass time 13.66 +[Apr 25, 2026 - 21:32:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 336 : backward pass time 11.53 +[Apr 25, 2026 - 21:32:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 336 : optimizer step time 0.98 +[Apr 25, 2026 - 21:32:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 336-------------------- +[Apr 25, 2026 - 21:32:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0370 iter count: 1.0 +[Apr 25, 2026 - 21:32:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0370 iter count: 1.0 +[Apr 25, 2026 - 21:32:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:32:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:33:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093868 43.158810 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.636230 70.640625 70.623047 +util 97.250000 99.000000 94.000000 +[Apr 25, 2026 - 21:33:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 337 : avg iteration time 47.04 seconds +[Apr 25, 2026 - 21:33:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 337 : data loading time 0.00 +[Apr 25, 2026 - 21:33:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 337 : avg forward pass time 11.49 +[Apr 25, 2026 - 21:33:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 337 : backward pass time 11.53 +[Apr 25, 2026 - 21:33:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 337 : optimizer step time 1.00 +[Apr 25, 2026 - 21:33:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 337-------------------- +[Apr 25, 2026 - 21:33:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0403 iter count: 1.0 +[Apr 25, 2026 - 21:33:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0403 iter count: 1.0 +[Apr 25, 2026 - 21:33:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:33:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:34:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093994 43.159313 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.631348 70.640625 70.603516 +util 98.250000 100.000000 95.000000 +[Apr 25, 2026 - 21:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 338 : avg iteration time 52.81 seconds +[Apr 25, 2026 - 21:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 338 : data loading time 0.00 +[Apr 25, 2026 - 21:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 338 : avg forward pass time 14.39 +[Apr 25, 2026 - 21:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 338 : backward pass time 11.52 +[Apr 25, 2026 - 21:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 338 : optimizer step time 0.97 +[Apr 25, 2026 - 21:34:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 338-------------------- +[Apr 25, 2026 - 21:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0349 iter count: 1.0 +[Apr 25, 2026 - 21:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0349 iter count: 1.0 +[Apr 25, 2026 - 21:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:34:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:34:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093994 43.159313 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.777832 70.835938 70.603516 +util 98.500000 99.000000 97.000000 +[Apr 25, 2026 - 21:34:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 339 : avg iteration time 49.89 seconds +[Apr 25, 2026 - 21:34:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 339 : data loading time 0.00 +[Apr 25, 2026 - 21:34:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 339 : avg forward pass time 12.93 +[Apr 25, 2026 - 21:34:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 339 : backward pass time 11.54 +[Apr 25, 2026 - 21:34:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 339 : optimizer step time 0.98 +[Apr 25, 2026 - 21:34:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 339-------------------- +[Apr 25, 2026 - 21:34:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0366 iter count: 1.0 +[Apr 25, 2026 - 21:34:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0366 iter count: 1.0 +[Apr 25, 2026 - 21:34:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:34:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.02 seconds +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093631 43.157841 43.004677 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.501953 135.515625 135.460938 +util 72.500000 76.000000 66.000000 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 340 : avg iteration time 169.79 seconds +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 340 : data loading time 0.00 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 340 : avg forward pass time 75.60 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 340 : backward pass time 9.15 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 340 : optimizer step time 0.34 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 340-------------------- +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 24618.3969 iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4716 iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 24618.3969iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.2492iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.3433 iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.6708 iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 49767.8984iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 108.4735iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 622566.8125iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4716iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.8209 iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 15396.1152iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.6340 iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 78271.0391iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.1040 iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 3670.2026iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.6015 iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1343.4257iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0375 iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:37:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 21:38:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093754 43.158340 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.650879 70.660156 70.623047 +util 95.500000 99.000000 92.000000 +[Apr 25, 2026 - 21:38:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 341 : avg iteration time 48.44 seconds +[Apr 25, 2026 - 21:38:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 341 : data loading time 0.00 +[Apr 25, 2026 - 21:38:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 341 : avg forward pass time 12.20 +[Apr 25, 2026 - 21:38:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 341 : backward pass time 11.53 +[Apr 25, 2026 - 21:38:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 341 : optimizer step time 0.98 +[Apr 25, 2026 - 21:38:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 341-------------------- +[Apr 25, 2026 - 21:38:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0321 iter count: 1.0 +[Apr 25, 2026 - 21:38:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0321 iter count: 1.0 +[Apr 25, 2026 - 21:38:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:38:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:39:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093942 43.159092 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.661133 70.681641 70.603516 +util 95.000000 100.000000 91.000000 +[Apr 25, 2026 - 21:39:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 342 : avg iteration time 47.00 seconds +[Apr 25, 2026 - 21:39:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 342 : data loading time 0.00 +[Apr 25, 2026 - 21:39:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 342 : avg forward pass time 11.46 +[Apr 25, 2026 - 21:39:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 342 : backward pass time 11.54 +[Apr 25, 2026 - 21:39:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 342 : optimizer step time 0.98 +[Apr 25, 2026 - 21:39:15 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 342-------------------- +[Apr 25, 2026 - 21:39:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0370 iter count: 1.0 +[Apr 25, 2026 - 21:39:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0370 iter count: 1.0 +[Apr 25, 2026 - 21:39:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:39:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:40:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094006 43.159344 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.690430 70.720703 70.603516 +util 96.000000 100.000000 92.000000 +[Apr 25, 2026 - 21:40:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 343 : avg iteration time 49.85 seconds +[Apr 25, 2026 - 21:40:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 343 : data loading time 0.00 +[Apr 25, 2026 - 21:40:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 343 : avg forward pass time 12.90 +[Apr 25, 2026 - 21:40:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 343 : backward pass time 11.54 +[Apr 25, 2026 - 21:40:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 343 : optimizer step time 0.97 +[Apr 25, 2026 - 21:40:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 343-------------------- +[Apr 25, 2026 - 21:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0345 iter count: 1.0 +[Apr 25, 2026 - 21:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0345 iter count: 1.0 +[Apr 25, 2026 - 21:40:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:40:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:40:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094072 43.159592 43.004700 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.602539 70.603516 70.601562 +util 96.250000 99.000000 93.000000 +[Apr 25, 2026 - 21:40:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 344 : avg iteration time 52.98 seconds +[Apr 25, 2026 - 21:40:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 344 : data loading time 0.00 +[Apr 25, 2026 - 21:40:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 344 : avg forward pass time 14.47 +[Apr 25, 2026 - 21:40:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 344 : backward pass time 11.52 +[Apr 25, 2026 - 21:40:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 344 : optimizer step time 0.98 +[Apr 25, 2026 - 21:40:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 344-------------------- +[Apr 25, 2026 - 21:40:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0284 iter count: 1.0 +[Apr 25, 2026 - 21:40:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0284 iter count: 1.0 +[Apr 25, 2026 - 21:40:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:40:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093572 43.157589 43.004677 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.522461 135.537109 135.480469 +util 73.500000 77.000000 67.000000 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 345 : avg iteration time 169.55 seconds +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 345 : data loading time 0.00 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 345 : avg forward pass time 75.51 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 345 : backward pass time 9.16 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 345 : optimizer step time 0.34 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 345-------------------- +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 86214.0933 iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4528 iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 86214.0933iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4026iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5597 iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.2660 iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 226091.5312iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.0630 iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2047271.8750iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4528iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.4344 iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 130.1690iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.2179 iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 623.8400iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.5592 iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1234.4908iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 1.6754 iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 28.5255 iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0323 iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:43:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 21:44:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093631 43.157837 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.650879 70.681641 70.640625 +util 97.500000 100.000000 92.000000 +[Apr 25, 2026 - 21:44:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 346 : avg iteration time 54.35 seconds +[Apr 25, 2026 - 21:44:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 346 : data loading time 0.00 +[Apr 25, 2026 - 21:44:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 346 : avg forward pass time 15.13 +[Apr 25, 2026 - 21:44:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 346 : backward pass time 11.54 +[Apr 25, 2026 - 21:44:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 346 : optimizer step time 0.99 +[Apr 25, 2026 - 21:44:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 346-------------------- +[Apr 25, 2026 - 21:44:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0385 iter count: 1.0 +[Apr 25, 2026 - 21:44:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0385 iter count: 1.0 +[Apr 25, 2026 - 21:44:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:44:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:45:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093945 43.159096 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.597168 70.601562 70.583984 +util 96.500000 100.000000 93.000000 +[Apr 25, 2026 - 21:45:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 347 : avg iteration time 49.94 seconds +[Apr 25, 2026 - 21:45:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 347 : data loading time 0.00 +[Apr 25, 2026 - 21:45:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 347 : avg forward pass time 12.94 +[Apr 25, 2026 - 21:45:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 347 : backward pass time 11.55 +[Apr 25, 2026 - 21:45:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 347 : optimizer step time 0.98 +[Apr 25, 2026 - 21:45:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 347-------------------- +[Apr 25, 2026 - 21:45:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0389 iter count: 1.0 +[Apr 25, 2026 - 21:45:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0389 iter count: 1.0 +[Apr 25, 2026 - 21:45:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:45:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:46:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094012 43.159351 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.597168 70.601562 70.583984 +util 96.500000 99.000000 91.000000 +[Apr 25, 2026 - 21:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 348 : avg iteration time 52.85 seconds +[Apr 25, 2026 - 21:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 348 : data loading time 0.00 +[Apr 25, 2026 - 21:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 348 : avg forward pass time 14.38 +[Apr 25, 2026 - 21:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 348 : backward pass time 11.55 +[Apr 25, 2026 - 21:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 348 : optimizer step time 1.00 +[Apr 25, 2026 - 21:46:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 348-------------------- +[Apr 25, 2026 - 21:46:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0365 iter count: 1.0 +[Apr 25, 2026 - 21:46:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0365 iter count: 1.0 +[Apr 25, 2026 - 21:46:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:46:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:47:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093965 43.159153 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.597168 70.601562 70.583984 +util 98.500000 100.000000 96.000000 +[Apr 25, 2026 - 21:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 349 : avg iteration time 47.06 seconds +[Apr 25, 2026 - 21:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 349 : data loading time 0.00 +[Apr 25, 2026 - 21:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 349 : avg forward pass time 11.50 +[Apr 25, 2026 - 21:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 349 : backward pass time 11.53 +[Apr 25, 2026 - 21:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 349 : optimizer step time 0.98 +[Apr 25, 2026 - 21:47:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 349-------------------- +[Apr 25, 2026 - 21:47:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0391 iter count: 1.0 +[Apr 25, 2026 - 21:47:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0391 iter count: 1.0 +[Apr 25, 2026 - 21:47:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:47:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093804 43.158543 43.004677 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.521973 135.537109 135.480469 +util 67.750000 74.000000 63.000000 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 350 : avg iteration time 170.09 seconds +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 350 : data loading time 0.00 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 350 : avg forward pass time 75.77 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 350 : backward pass time 9.15 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 350 : optimizer step time 0.34 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 350-------------------- +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 1012832.8179 iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5202 iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 1012832.8179iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3294iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.0884 iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 9.0532 iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 2400504.2500iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 8.8188 iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 73033312.0000iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5202iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.8970 iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 2426.2258iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.5920 iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 71976.6641iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 1.8141 iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 37.6442 iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.4510 iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 7346.7812iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0345 iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:50:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 21:50:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093801 43.158527 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.812012 70.875000 70.623047 +util 97.000000 99.000000 92.000000 +[Apr 25, 2026 - 21:50:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 351 : avg iteration time 51.26 seconds +[Apr 25, 2026 - 21:50:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 351 : data loading time 0.00 +[Apr 25, 2026 - 21:50:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 351 : avg forward pass time 13.61 +[Apr 25, 2026 - 21:50:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 351 : backward pass time 11.53 +[Apr 25, 2026 - 21:50:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 351 : optimizer step time 0.98 +[Apr 25, 2026 - 21:50:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 351-------------------- +[Apr 25, 2026 - 21:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0348 iter count: 1.0 +[Apr 25, 2026 - 21:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0348 iter count: 1.0 +[Apr 25, 2026 - 21:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:50:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:51:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093988 43.159271 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.777832 70.835938 70.603516 +util 94.500000 99.000000 91.000000 +[Apr 25, 2026 - 21:51:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 352 : avg iteration time 49.91 seconds +[Apr 25, 2026 - 21:51:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 352 : data loading time 0.00 +[Apr 25, 2026 - 21:51:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 352 : avg forward pass time 12.93 +[Apr 25, 2026 - 21:51:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 352 : backward pass time 11.53 +[Apr 25, 2026 - 21:51:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 352 : optimizer step time 0.98 +[Apr 25, 2026 - 21:51:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 352-------------------- +[Apr 25, 2026 - 21:51:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0343 iter count: 1.0 +[Apr 25, 2026 - 21:51:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0343 iter count: 1.0 +[Apr 25, 2026 - 21:51:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:51:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:52:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094000 43.159271 43.004700 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.768066 70.816406 70.623047 +util 95.500000 100.000000 92.000000 +[Apr 25, 2026 - 21:52:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 353 : avg iteration time 52.81 seconds +[Apr 25, 2026 - 21:52:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 353 : data loading time 0.00 +[Apr 25, 2026 - 21:52:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 353 : avg forward pass time 14.38 +[Apr 25, 2026 - 21:52:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 353 : backward pass time 11.55 +[Apr 25, 2026 - 21:52:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 353 : optimizer step time 0.98 +[Apr 25, 2026 - 21:52:36 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 353-------------------- +[Apr 25, 2026 - 21:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0320 iter count: 1.0 +[Apr 25, 2026 - 21:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0320 iter count: 1.0 +[Apr 25, 2026 - 21:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:52:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:53:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094009 43.159294 43.004700 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.631348 70.640625 70.603516 +util 95.000000 99.000000 91.000000 +[Apr 25, 2026 - 21:53:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 354 : avg iteration time 47.14 seconds +[Apr 25, 2026 - 21:53:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 354 : data loading time 0.00 +[Apr 25, 2026 - 21:53:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 354 : avg forward pass time 11.53 +[Apr 25, 2026 - 21:53:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 354 : backward pass time 11.57 +[Apr 25, 2026 - 21:53:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 354 : optimizer step time 0.99 +[Apr 25, 2026 - 21:53:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 354-------------------- +[Apr 25, 2026 - 21:53:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0359 iter count: 1.0 +[Apr 25, 2026 - 21:53:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0359 iter count: 1.0 +[Apr 25, 2026 - 21:53:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:53:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093715 43.158150 43.004711 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.479004 135.480469 135.478516 +util 71.250000 75.000000 65.000000 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 355 : avg iteration time 172.96 seconds +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 355 : data loading time 0.00 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 355 : avg forward pass time 77.22 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 355 : backward pass time 9.13 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 355 : optimizer step time 0.34 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 355-------------------- +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 43360.0836 iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4688 iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 43360.0836iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3478iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.5897 iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.2679 iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 126743.3125iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 24.0345iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2055204.2500iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4688iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.0749 iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 25586.3086iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.4163 iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 927.5938iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.8209 iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 2083.2920iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.7493 iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 98581.5156iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0342 iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:56:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 21:57:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093841 43.158653 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.261230 73.357422 70.562500 +util 95.500000 99.000000 92.000000 +[Apr 25, 2026 - 21:57:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 356 : avg iteration time 51.49 seconds +[Apr 25, 2026 - 21:57:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 356 : data loading time 0.00 +[Apr 25, 2026 - 21:57:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 356 : avg forward pass time 13.72 +[Apr 25, 2026 - 21:57:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 356 : backward pass time 11.56 +[Apr 25, 2026 - 21:57:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 356 : optimizer step time 0.97 +[Apr 25, 2026 - 21:57:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 356-------------------- +[Apr 25, 2026 - 21:57:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0352 iter count: 1.0 +[Apr 25, 2026 - 21:57:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0352 iter count: 1.0 +[Apr 25, 2026 - 21:57:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:57:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:57:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093841 43.158653 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.402832 73.455078 70.718750 +util 95.500000 99.000000 92.000000 +[Apr 25, 2026 - 21:57:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 357 : avg iteration time 47.00 seconds +[Apr 25, 2026 - 21:57:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 357 : data loading time 0.00 +[Apr 25, 2026 - 21:57:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 357 : avg forward pass time 11.47 +[Apr 25, 2026 - 21:57:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 357 : backward pass time 11.52 +[Apr 25, 2026 - 21:57:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 357 : optimizer step time 0.97 +[Apr 25, 2026 - 21:57:54 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 357-------------------- +[Apr 25, 2026 - 21:57:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0345 iter count: 1.0 +[Apr 25, 2026 - 21:57:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0345 iter count: 1.0 +[Apr 25, 2026 - 21:57:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:57:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:58:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093832 43.158653 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.339355 73.494141 70.621094 +util 98.000000 100.000000 94.000000 +[Apr 25, 2026 - 21:58:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 358 : avg iteration time 47.30 seconds +[Apr 25, 2026 - 21:58:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 358 : data loading time 0.00 +[Apr 25, 2026 - 21:58:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 358 : avg forward pass time 11.59 +[Apr 25, 2026 - 21:58:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 358 : backward pass time 11.52 +[Apr 25, 2026 - 21:58:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 358 : optimizer step time 1.00 +[Apr 25, 2026 - 21:58:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 358-------------------- +[Apr 25, 2026 - 21:58:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0316 iter count: 1.0 +[Apr 25, 2026 - 21:58:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0316 iter count: 1.0 +[Apr 25, 2026 - 21:58:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:58:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 21:59:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094033 43.159401 43.004723 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.222168 73.259766 70.542969 +util 97.000000 99.000000 92.000000 +[Apr 25, 2026 - 21:59:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 359 : avg iteration time 47.18 seconds +[Apr 25, 2026 - 21:59:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 359 : data loading time 0.00 +[Apr 25, 2026 - 21:59:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 359 : avg forward pass time 11.54 +[Apr 25, 2026 - 21:59:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 359 : backward pass time 11.54 +[Apr 25, 2026 - 21:59:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 359 : optimizer step time 1.00 +[Apr 25, 2026 - 21:59:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 359-------------------- +[Apr 25, 2026 - 21:59:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0389 iter count: 1.0 +[Apr 25, 2026 - 21:59:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0389 iter count: 1.0 +[Apr 25, 2026 - 21:59:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 21:59:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093747 43.158291 43.004677 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.541992 135.576172 135.439453 +util 68.500000 77.000000 63.000000 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 360 : avg iteration time 172.89 seconds +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 360 : data loading time 0.00 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 360 : avg forward pass time 77.18 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 360 : backward pass time 9.14 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 360 : optimizer step time 0.34 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 360-------------------- +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 171679.4968 iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4785 iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 171679.4968iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3638iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6459 iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.6674 iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 260068.5781iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.6393 iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 4568876.0000iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4785iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 7.6674 iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 4568876.0000iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.5679 iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 1256.2102iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 7.1297 iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1558827.1250iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.8752 iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 936956.3750iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0335 iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:02:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 22:03:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093808 43.158543 43.004681 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.489746 70.505859 70.484375 +util 98.000000 100.000000 94.000000 +[Apr 25, 2026 - 22:03:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 361 : avg iteration time 51.32 seconds +[Apr 25, 2026 - 22:03:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 361 : data loading time 0.00 +[Apr 25, 2026 - 22:03:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 361 : avg forward pass time 13.63 +[Apr 25, 2026 - 22:03:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 361 : backward pass time 11.54 +[Apr 25, 2026 - 22:03:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 361 : optimizer step time 0.97 +[Apr 25, 2026 - 22:03:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 361-------------------- +[Apr 25, 2026 - 22:03:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0410 iter count: 1.0 +[Apr 25, 2026 - 22:03:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0410 iter count: 1.0 +[Apr 25, 2026 - 22:03:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:03:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 22:04:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093804 43.158543 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.484863 70.603516 70.445312 +util 98.500000 100.000000 96.000000 +[Apr 25, 2026 - 22:04:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 362 : avg iteration time 47.02 seconds +[Apr 25, 2026 - 22:04:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 362 : data loading time 0.00 +[Apr 25, 2026 - 22:04:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 362 : avg forward pass time 11.47 +[Apr 25, 2026 - 22:04:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 362 : backward pass time 11.54 +[Apr 25, 2026 - 22:04:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 362 : optimizer step time 0.99 +[Apr 25, 2026 - 22:04:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 362-------------------- +[Apr 25, 2026 - 22:04:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0346 iter count: 1.0 +[Apr 25, 2026 - 22:04:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0346 iter count: 1.0 +[Apr 25, 2026 - 22:04:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:04:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:04:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093816 43.158543 43.004723 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.455566 70.486328 70.445312 +util 97.750000 99.000000 95.000000 +[Apr 25, 2026 - 22:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 363 : avg iteration time 49.91 seconds +[Apr 25, 2026 - 22:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 363 : data loading time 0.00 +[Apr 25, 2026 - 22:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 363 : avg forward pass time 12.91 +[Apr 25, 2026 - 22:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 363 : backward pass time 11.56 +[Apr 25, 2026 - 22:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 363 : optimizer step time 0.98 +[Apr 25, 2026 - 22:04:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 363-------------------- +[Apr 25, 2026 - 22:04:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0390 iter count: 1.0 +[Apr 25, 2026 - 22:04:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0390 iter count: 1.0 +[Apr 25, 2026 - 22:04:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:04:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:05:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093864 43.158783 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.484863 70.603516 70.445312 +util 94.250000 99.000000 90.000000 +[Apr 25, 2026 - 22:05:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 364 : avg iteration time 46.95 seconds +[Apr 25, 2026 - 22:05:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 364 : data loading time 0.00 +[Apr 25, 2026 - 22:05:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 364 : avg forward pass time 11.45 +[Apr 25, 2026 - 22:05:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 364 : backward pass time 11.53 +[Apr 25, 2026 - 22:05:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 364 : optimizer step time 0.99 +[Apr 25, 2026 - 22:05:37 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 364-------------------- +[Apr 25, 2026 - 22:05:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0361 iter count: 1.0 +[Apr 25, 2026 - 22:05:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0361 iter count: 1.0 +[Apr 25, 2026 - 22:05:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:05:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093804 43.158524 43.004696 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.518066 135.537109 135.460938 +util 73.750000 77.000000 67.000000 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 365 : avg iteration time 172.55 seconds +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 365 : data loading time 0.00 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 365 : avg forward pass time 77.00 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 365 : backward pass time 9.13 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 365 : optimizer step time 0.34 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 365-------------------- +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 183749.7578 iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6175 iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 183749.7578iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.8293iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.0059 iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.3689 iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 319460.3125iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 7.4774 iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2514820.2500iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6175iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.4688 iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 56252.7773iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.3911 iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 48159.1602iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.3237 iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 5695.0474iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.9540 iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 20089.8770iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0358 iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 22:09:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093930 43.159031 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.680176 70.798828 70.640625 +util 98.500000 99.000000 97.000000 +[Apr 25, 2026 - 22:09:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 366 : avg iteration time 51.40 seconds +[Apr 25, 2026 - 22:09:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 366 : data loading time 0.00 +[Apr 25, 2026 - 22:09:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 366 : avg forward pass time 13.65 +[Apr 25, 2026 - 22:09:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 366 : backward pass time 11.57 +[Apr 25, 2026 - 22:09:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 366 : optimizer step time 0.99 +[Apr 25, 2026 - 22:09:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 366-------------------- +[Apr 25, 2026 - 22:09:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0358 iter count: 1.0 +[Apr 25, 2026 - 22:09:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0358 iter count: 1.0 +[Apr 25, 2026 - 22:09:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:09:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:10:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093925 43.159027 43.004677 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.685059 70.699219 70.642578 +util 95.500000 99.000000 92.000000 +[Apr 25, 2026 - 22:10:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 367 : avg iteration time 49.87 seconds +[Apr 25, 2026 - 22:10:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 367 : data loading time 0.00 +[Apr 25, 2026 - 22:10:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 367 : avg forward pass time 12.90 +[Apr 25, 2026 - 22:10:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 367 : backward pass time 11.55 +[Apr 25, 2026 - 22:10:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 367 : optimizer step time 0.98 +[Apr 25, 2026 - 22:10:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 367-------------------- +[Apr 25, 2026 - 22:10:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0367 iter count: 1.0 +[Apr 25, 2026 - 22:10:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0367 iter count: 1.0 +[Apr 25, 2026 - 22:10:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:10:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:11:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093932 43.159027 43.004684 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.543457 70.779297 70.464844 +util 96.750000 99.000000 91.000000 +[Apr 25, 2026 - 22:11:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 368 : avg iteration time 52.86 seconds +[Apr 25, 2026 - 22:11:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 368 : data loading time 0.00 +[Apr 25, 2026 - 22:11:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 368 : avg forward pass time 14.40 +[Apr 25, 2026 - 22:11:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 368 : backward pass time 11.54 +[Apr 25, 2026 - 22:11:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 368 : optimizer step time 0.98 +[Apr 25, 2026 - 22:11:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 368-------------------- +[Apr 25, 2026 - 22:11:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0362 iter count: 1.0 +[Apr 25, 2026 - 22:11:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0362 iter count: 1.0 +[Apr 25, 2026 - 22:11:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:11:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:11:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093929 43.159027 43.004684 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.729004 70.759766 70.718750 +util 94.750000 99.000000 91.000000 +[Apr 25, 2026 - 22:11:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 369 : avg iteration time 52.93 seconds +[Apr 25, 2026 - 22:11:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 369 : data loading time 0.00 +[Apr 25, 2026 - 22:11:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 369 : avg forward pass time 14.44 +[Apr 25, 2026 - 22:11:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 369 : backward pass time 11.54 +[Apr 25, 2026 - 22:11:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 369 : optimizer step time 0.98 +[Apr 25, 2026 - 22:11:57 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 369-------------------- +[Apr 25, 2026 - 22:11:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0386 iter count: 1.0 +[Apr 25, 2026 - 22:11:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0386 iter count: 1.0 +[Apr 25, 2026 - 22:11:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:11:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093929 43.159027 43.004684 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.595703 135.615234 135.539062 +util 69.750000 73.000000 64.000000 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 370 : avg iteration time 175.89 seconds +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 370 : data loading time 0.00 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 370 : avg forward pass time 78.66 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 370 : backward pass time 9.18 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 370 : optimizer step time 0.34 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 370-------------------- +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 420943.2754 iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5302 iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 420943.2754iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 5.0006iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.9905 iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.0399 iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 686490.0625iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 7.2496 iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 9625088.0000iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5302iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.0969 iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 489.7287iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 7.4220 iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2796664.2500iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.3704 iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 6253.1040iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.9151 iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1014864.7500iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0356 iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 22:15:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093937 43.159027 43.004696 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.528809 70.603516 70.503906 +util 97.750000 99.000000 94.000000 +[Apr 25, 2026 - 22:15:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 371 : avg iteration time 48.41 seconds +[Apr 25, 2026 - 22:15:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 371 : data loading time 0.00 +[Apr 25, 2026 - 22:15:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 371 : avg forward pass time 12.17 +[Apr 25, 2026 - 22:15:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 371 : backward pass time 11.53 +[Apr 25, 2026 - 22:15:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 371 : optimizer step time 0.99 +[Apr 25, 2026 - 22:15:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 371-------------------- +[Apr 25, 2026 - 22:15:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0419 iter count: 1.0 +[Apr 25, 2026 - 22:15:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0419 iter count: 1.0 +[Apr 25, 2026 - 22:15:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:15:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:16:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093938 43.159027 43.004719 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.694824 70.718750 70.623047 +util 97.750000 100.000000 93.000000 +[Apr 25, 2026 - 22:16:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 372 : avg iteration time 46.99 seconds +[Apr 25, 2026 - 22:16:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 372 : data loading time 0.00 +[Apr 25, 2026 - 22:16:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 372 : avg forward pass time 11.48 +[Apr 25, 2026 - 22:16:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 372 : backward pass time 11.52 +[Apr 25, 2026 - 22:16:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 372 : optimizer step time 0.98 +[Apr 25, 2026 - 22:16:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 372-------------------- +[Apr 25, 2026 - 22:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0360 iter count: 1.0 +[Apr 25, 2026 - 22:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0360 iter count: 1.0 +[Apr 25, 2026 - 22:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:16:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:17:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093936 43.159027 43.004696 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.636230 70.640625 70.623047 +util 98.500000 99.000000 97.000000 +[Apr 25, 2026 - 22:17:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 373 : avg iteration time 52.85 seconds +[Apr 25, 2026 - 22:17:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 373 : data loading time 0.00 +[Apr 25, 2026 - 22:17:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 373 : avg forward pass time 14.38 +[Apr 25, 2026 - 22:17:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 373 : backward pass time 11.55 +[Apr 25, 2026 - 22:17:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 373 : optimizer step time 0.98 +[Apr 25, 2026 - 22:17:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 373-------------------- +[Apr 25, 2026 - 22:17:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0380 iter count: 1.0 +[Apr 25, 2026 - 22:17:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0380 iter count: 1.0 +[Apr 25, 2026 - 22:17:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:17:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:18:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093932 43.159027 43.004696 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.616699 70.621094 70.603516 +util 98.250000 100.000000 95.000000 +[Apr 25, 2026 - 22:18:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 374 : avg iteration time 49.97 seconds +[Apr 25, 2026 - 22:18:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 374 : data loading time 0.00 +[Apr 25, 2026 - 22:18:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 374 : avg forward pass time 12.94 +[Apr 25, 2026 - 22:18:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 374 : backward pass time 11.54 +[Apr 25, 2026 - 22:18:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 374 : optimizer step time 1.00 +[Apr 25, 2026 - 22:18:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 374-------------------- +[Apr 25, 2026 - 22:18:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0388 iter count: 1.0 +[Apr 25, 2026 - 22:18:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0388 iter count: 1.0 +[Apr 25, 2026 - 22:18:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:18:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.02 seconds +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093941 43.159027 43.004707 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.526367 135.537109 135.498047 +util 70.000000 76.000000 65.000000 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 375 : avg iteration time 172.94 seconds +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 375 : data loading time 0.00 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 375 : avg forward pass time 77.17 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 375 : backward pass time 9.16 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 375 : optimizer step time 0.34 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 375-------------------- +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 525271.1526 iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4422 iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 525271.1526iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3295iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.1710 iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.7671 iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 1597607.1250iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 10.4030iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 41206840.0000iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4422iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.7725 iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1891.4144iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.2102 iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 33536.6758iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.3598 iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 828.5466iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.4086 iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 49867.6211iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0350 iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:21:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 22:21:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093935 43.159027 43.004707 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.334473 73.474609 70.621094 +util 95.750000 99.000000 92.000000 +[Apr 25, 2026 - 22:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 376 : avg iteration time 51.33 seconds +[Apr 25, 2026 - 22:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 376 : data loading time 0.00 +[Apr 25, 2026 - 22:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 376 : avg forward pass time 13.64 +[Apr 25, 2026 - 22:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 376 : backward pass time 11.53 +[Apr 25, 2026 - 22:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 376 : optimizer step time 0.99 +[Apr 25, 2026 - 22:21:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 376-------------------- +[Apr 25, 2026 - 22:21:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0339 iter count: 1.0 +[Apr 25, 2026 - 22:21:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0339 iter count: 1.0 +[Apr 25, 2026 - 22:21:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:21:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:22:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093935 43.159027 43.004707 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.324707 73.494141 70.601562 +util 96.750000 99.000000 91.000000 +[Apr 25, 2026 - 22:22:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 377 : avg iteration time 49.97 seconds +[Apr 25, 2026 - 22:22:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 377 : data loading time 0.00 +[Apr 25, 2026 - 22:22:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 377 : avg forward pass time 12.95 +[Apr 25, 2026 - 22:22:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 377 : backward pass time 11.54 +[Apr 25, 2026 - 22:22:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 377 : optimizer step time 0.98 +[Apr 25, 2026 - 22:22:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 377-------------------- +[Apr 25, 2026 - 22:22:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0339 iter count: 1.0 +[Apr 25, 2026 - 22:22:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0339 iter count: 1.0 +[Apr 25, 2026 - 22:22:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:22:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:23:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093892 43.158852 43.004707 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.373535 73.513672 70.660156 +util 96.000000 100.000000 92.000000 +[Apr 25, 2026 - 22:23:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 378 : avg iteration time 50.04 seconds +[Apr 25, 2026 - 22:23:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 378 : data loading time 0.00 +[Apr 25, 2026 - 22:23:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 378 : avg forward pass time 13.00 +[Apr 25, 2026 - 22:23:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 378 : backward pass time 11.53 +[Apr 25, 2026 - 22:23:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 378 : optimizer step time 0.99 +[Apr 25, 2026 - 22:23:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 378-------------------- +[Apr 25, 2026 - 22:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0303 iter count: 1.0 +[Apr 25, 2026 - 22:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0303 iter count: 1.0 +[Apr 25, 2026 - 22:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:23:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:24:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093897 43.158867 43.004707 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.378418 73.474609 70.679688 +util 97.250000 100.000000 92.000000 +[Apr 25, 2026 - 22:24:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 379 : avg iteration time 49.89 seconds +[Apr 25, 2026 - 22:24:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 379 : data loading time 0.00 +[Apr 25, 2026 - 22:24:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 379 : avg forward pass time 12.91 +[Apr 25, 2026 - 22:24:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 379 : backward pass time 11.54 +[Apr 25, 2026 - 22:24:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 379 : optimizer step time 0.98 +[Apr 25, 2026 - 22:24:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 379-------------------- +[Apr 25, 2026 - 22:24:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0313 iter count: 1.0 +[Apr 25, 2026 - 22:24:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0313 iter count: 1.0 +[Apr 25, 2026 - 22:24:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:24:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094036 43.158756 43.004921 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.522461 135.537109 135.480469 +util 70.250000 77.000000 65.000000 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 380 : avg iteration time 173.17 seconds +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 380 : data loading time 0.00 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 380 : avg forward pass time 77.29 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 380 : backward pass time 9.17 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 380 : optimizer step time 0.34 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 380-------------------- +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 80540.9434 iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4906 iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 80540.9434iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.8219iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.8656 iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.1374 iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 215505.3750iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 41.7259iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1582987.1250iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4906iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.4444 iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 7250.1729iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.2389 iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 262435.6562iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.5392 iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1186.0190iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.0011 iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 2987.2473iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0341 iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:27:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 22:28:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094036 43.158756 43.004932 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.479980 70.583984 70.445312 +util 98.250000 100.000000 95.000000 +[Apr 25, 2026 - 22:28:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 381 : avg iteration time 51.28 seconds +[Apr 25, 2026 - 22:28:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 381 : data loading time 0.00 +[Apr 25, 2026 - 22:28:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 381 : avg forward pass time 13.62 +[Apr 25, 2026 - 22:28:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 381 : backward pass time 11.52 +[Apr 25, 2026 - 22:28:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 381 : optimizer step time 1.00 +[Apr 25, 2026 - 22:28:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 381-------------------- +[Apr 25, 2026 - 22:28:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0263 iter count: 1.0 +[Apr 25, 2026 - 22:28:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0263 iter count: 1.0 +[Apr 25, 2026 - 22:28:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:28:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:29:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094343 43.159996 43.004921 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.538574 70.701172 70.484375 +util 98.500000 100.000000 96.000000 +[Apr 25, 2026 - 22:29:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 382 : avg iteration time 52.80 seconds +[Apr 25, 2026 - 22:29:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 382 : data loading time 0.00 +[Apr 25, 2026 - 22:29:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 382 : avg forward pass time 14.39 +[Apr 25, 2026 - 22:29:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 382 : backward pass time 11.52 +[Apr 25, 2026 - 22:29:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 382 : optimizer step time 0.99 +[Apr 25, 2026 - 22:29:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 382-------------------- +[Apr 25, 2026 - 22:29:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0219 iter count: 1.0 +[Apr 25, 2026 - 22:29:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0219 iter count: 1.0 +[Apr 25, 2026 - 22:29:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:29:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:29:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094415 43.160248 43.004921 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.455566 70.544922 70.425781 +util 98.000000 100.000000 94.000000 +[Apr 25, 2026 - 22:29:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 383 : avg iteration time 46.98 seconds +[Apr 25, 2026 - 22:29:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 383 : data loading time 0.00 +[Apr 25, 2026 - 22:29:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 383 : avg forward pass time 11.47 +[Apr 25, 2026 - 22:29:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 383 : backward pass time 11.50 +[Apr 25, 2026 - 22:29:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 383 : optimizer step time 0.99 +[Apr 25, 2026 - 22:29:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 383-------------------- +[Apr 25, 2026 - 22:29:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0253 iter count: 1.0 +[Apr 25, 2026 - 22:29:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0253 iter count: 1.0 +[Apr 25, 2026 - 22:29:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:29:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:30:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094418 43.160248 43.004925 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.689941 70.699219 70.662109 +util 98.500000 100.000000 96.000000 +[Apr 25, 2026 - 22:30:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 384 : avg iteration time 52.83 seconds +[Apr 25, 2026 - 22:30:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 384 : data loading time 0.00 +[Apr 25, 2026 - 22:30:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 384 : avg forward pass time 14.38 +[Apr 25, 2026 - 22:30:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 384 : backward pass time 11.53 +[Apr 25, 2026 - 22:30:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 384 : optimizer step time 1.00 +[Apr 25, 2026 - 22:30:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 384-------------------- +[Apr 25, 2026 - 22:30:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0234 iter count: 1.0 +[Apr 25, 2026 - 22:30:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0234 iter count: 1.0 +[Apr 25, 2026 - 22:30:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:30:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094428 43.160309 43.004932 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.522949 135.537109 135.480469 +util 69.250000 78.000000 64.000000 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 385 : avg iteration time 172.86 seconds +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 385 : data loading time 0.00 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 385 : avg forward pass time 77.16 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 385 : backward pass time 9.14 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 385 : optimizer step time 0.34 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 385-------------------- +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 71819.1450 iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5405 iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 71819.1450iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.7903iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.2967 iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.8754 iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 143415.7031iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 98.8328iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 937316.5625iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5405iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.3887 iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 877.6990iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.7282 iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 1730.8591iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 6.7566 iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 739178.1875iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.9297 iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 19136.5957iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0238 iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:33:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 22:34:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094443 43.160385 43.004932 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.694824 70.740234 70.679688 +util 98.500000 100.000000 96.000000 +[Apr 25, 2026 - 22:34:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 386 : avg iteration time 54.26 seconds +[Apr 25, 2026 - 22:34:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 386 : data loading time 0.00 +[Apr 25, 2026 - 22:34:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 386 : avg forward pass time 15.09 +[Apr 25, 2026 - 22:34:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 386 : backward pass time 11.59 +[Apr 25, 2026 - 22:34:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 386 : optimizer step time 0.97 +[Apr 25, 2026 - 22:34:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 386-------------------- +[Apr 25, 2026 - 22:34:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0283 iter count: 1.0 +[Apr 25, 2026 - 22:34:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0283 iter count: 1.0 +[Apr 25, 2026 - 22:34:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:34:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:35:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094670 43.161282 43.004921 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.602051 70.662109 70.582031 +util 97.250000 100.000000 92.000000 +[Apr 25, 2026 - 22:35:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 387 : avg iteration time 50.02 seconds +[Apr 25, 2026 - 22:35:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 387 : data loading time 0.00 +[Apr 25, 2026 - 22:35:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 387 : avg forward pass time 12.96 +[Apr 25, 2026 - 22:35:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 387 : backward pass time 11.58 +[Apr 25, 2026 - 22:35:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 387 : optimizer step time 0.98 +[Apr 25, 2026 - 22:35:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 387-------------------- +[Apr 25, 2026 - 22:35:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0278 iter count: 1.0 +[Apr 25, 2026 - 22:35:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0278 iter count: 1.0 +[Apr 25, 2026 - 22:35:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:35:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:36:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094661 43.161266 43.004921 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.602051 70.603516 70.601562 +util 97.750000 99.000000 95.000000 +[Apr 25, 2026 - 22:36:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 388 : avg iteration time 49.97 seconds +[Apr 25, 2026 - 22:36:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 388 : data loading time 0.00 +[Apr 25, 2026 - 22:36:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 388 : avg forward pass time 12.94 +[Apr 25, 2026 - 22:36:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 388 : backward pass time 11.57 +[Apr 25, 2026 - 22:36:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 388 : optimizer step time 0.98 +[Apr 25, 2026 - 22:36:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 388-------------------- +[Apr 25, 2026 - 22:36:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0240 iter count: 1.0 +[Apr 25, 2026 - 22:36:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0240 iter count: 1.0 +[Apr 25, 2026 - 22:36:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:36:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:37:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094681 43.161346 43.004921 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.694824 70.718750 70.623047 +util 98.750000 100.000000 97.000000 +[Apr 25, 2026 - 22:37:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 389 : avg iteration time 52.89 seconds +[Apr 25, 2026 - 22:37:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 389 : data loading time 0.00 +[Apr 25, 2026 - 22:37:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 389 : avg forward pass time 14.40 +[Apr 25, 2026 - 22:37:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 389 : backward pass time 11.56 +[Apr 25, 2026 - 22:37:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 389 : optimizer step time 0.98 +[Apr 25, 2026 - 22:37:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 389-------------------- +[Apr 25, 2026 - 22:37:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0233 iter count: 1.0 +[Apr 25, 2026 - 22:37:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0233 iter count: 1.0 +[Apr 25, 2026 - 22:37:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:37:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094484 43.160538 43.004921 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.488770 135.498047 135.460938 +util 68.250000 72.000000 62.000000 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 390 : avg iteration time 178.74 seconds +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 390 : data loading time 0.00 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 390 : avg forward pass time 80.09 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 390 : backward pass time 9.17 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 390 : optimizer step time 0.34 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 390-------------------- +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 606199.7959 iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5900 iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 606199.7959iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.7597iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.3957 iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.7159 iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 1358614.1250iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.2066 iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 37202364.0000iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5900iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.6914 iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 11881.7295iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.5253 iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 62978.1328iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.7347 iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 237.2961iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.8601 iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 2253.5681iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0247 iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:40:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 22:40:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094506 43.160645 43.004921 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.553223 70.642578 70.523438 +util 98.750000 100.000000 97.000000 +[Apr 25, 2026 - 22:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 391 : avg iteration time 51.66 seconds +[Apr 25, 2026 - 22:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 391 : data loading time 0.00 +[Apr 25, 2026 - 22:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 391 : avg forward pass time 13.78 +[Apr 25, 2026 - 22:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 391 : backward pass time 11.53 +[Apr 25, 2026 - 22:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 391 : optimizer step time 0.99 +[Apr 25, 2026 - 22:40:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 391-------------------- +[Apr 25, 2026 - 22:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0266 iter count: 1.0 +[Apr 25, 2026 - 22:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0266 iter count: 1.0 +[Apr 25, 2026 - 22:40:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:40:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:41:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094514 43.160645 43.004921 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.680176 70.699219 70.623047 +util 96.500000 100.000000 91.000000 +[Apr 25, 2026 - 22:41:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 392 : avg iteration time 49.93 seconds +[Apr 25, 2026 - 22:41:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 392 : data loading time 0.00 +[Apr 25, 2026 - 22:41:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 392 : avg forward pass time 12.92 +[Apr 25, 2026 - 22:41:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 392 : backward pass time 11.54 +[Apr 25, 2026 - 22:41:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 392 : optimizer step time 0.99 +[Apr 25, 2026 - 22:41:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 392-------------------- +[Apr 25, 2026 - 22:41:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0271 iter count: 1.0 +[Apr 25, 2026 - 22:41:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0271 iter count: 1.0 +[Apr 25, 2026 - 22:41:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:41:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:42:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094505 43.160645 43.004921 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.665527 70.679688 70.623047 +util 97.750000 99.000000 95.000000 +[Apr 25, 2026 - 22:42:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 393 : avg iteration time 49.98 seconds +[Apr 25, 2026 - 22:42:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 393 : data loading time 0.00 +[Apr 25, 2026 - 22:42:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 393 : avg forward pass time 12.96 +[Apr 25, 2026 - 22:42:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 393 : backward pass time 11.53 +[Apr 25, 2026 - 22:42:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 393 : optimizer step time 0.98 +[Apr 25, 2026 - 22:42:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 393-------------------- +[Apr 25, 2026 - 22:42:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0265 iter count: 1.0 +[Apr 25, 2026 - 22:42:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0265 iter count: 1.0 +[Apr 25, 2026 - 22:42:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:42:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:43:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094563 43.160854 43.004921 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.738770 70.777344 70.623047 +util 98.500000 100.000000 95.000000 +[Apr 25, 2026 - 22:43:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 394 : avg iteration time 47.07 seconds +[Apr 25, 2026 - 22:43:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 394 : data loading time 0.00 +[Apr 25, 2026 - 22:43:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 394 : avg forward pass time 11.48 +[Apr 25, 2026 - 22:43:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 394 : backward pass time 11.58 +[Apr 25, 2026 - 22:43:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 394 : optimizer step time 0.98 +[Apr 25, 2026 - 22:43:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 394-------------------- +[Apr 25, 2026 - 22:43:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0280 iter count: 1.0 +[Apr 25, 2026 - 22:43:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0280 iter count: 1.0 +[Apr 25, 2026 - 22:43:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:43:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094320 43.159904 43.004921 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.584961 135.595703 135.556641 +util 67.250000 74.000000 62.000000 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 395 : avg iteration time 166.90 seconds +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 395 : data loading time 0.00 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 395 : avg forward pass time 74.18 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 395 : backward pass time 9.15 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 395 : optimizer step time 0.34 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 395-------------------- +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 384240.6865 iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5126 iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 384240.6865iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 5.0337iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.4842 iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.2874 iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 841508.6875iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 143.8022iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 15788839.0000iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5126iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.4969 iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 8053.1323iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.0086 iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 165580.8281iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.6655 iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 83368.8359iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.4465 iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 53799.4102iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0256 iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:46:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 22:46:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094327 43.159904 43.004921 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.528809 70.662109 70.484375 +util 95.500000 100.000000 92.000000 +[Apr 25, 2026 - 22:46:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 396 : avg iteration time 48.59 seconds +[Apr 25, 2026 - 22:46:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 396 : data loading time 0.00 +[Apr 25, 2026 - 22:46:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 396 : avg forward pass time 12.26 +[Apr 25, 2026 - 22:46:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 396 : backward pass time 11.53 +[Apr 25, 2026 - 22:46:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 396 : optimizer step time 1.00 +[Apr 25, 2026 - 22:46:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 396-------------------- +[Apr 25, 2026 - 22:46:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0287 iter count: 1.0 +[Apr 25, 2026 - 22:46:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0287 iter count: 1.0 +[Apr 25, 2026 - 22:46:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:46:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:47:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094365 43.160076 43.004921 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.768066 70.796875 70.681641 +util 96.000000 99.000000 92.000000 +[Apr 25, 2026 - 22:47:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 397 : avg iteration time 49.98 seconds +[Apr 25, 2026 - 22:47:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 397 : data loading time 0.00 +[Apr 25, 2026 - 22:47:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 397 : avg forward pass time 12.96 +[Apr 25, 2026 - 22:47:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 397 : backward pass time 11.51 +[Apr 25, 2026 - 22:47:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 397 : optimizer step time 1.00 +[Apr 25, 2026 - 22:47:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 397-------------------- +[Apr 25, 2026 - 22:47:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0282 iter count: 1.0 +[Apr 25, 2026 - 22:47:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0282 iter count: 1.0 +[Apr 25, 2026 - 22:47:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:47:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:48:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094364 43.160030 43.004921 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.650879 70.681641 70.640625 +util 97.750000 99.000000 95.000000 +[Apr 25, 2026 - 22:48:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 398 : avg iteration time 52.86 seconds +[Apr 25, 2026 - 22:48:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 398 : data loading time 0.00 +[Apr 25, 2026 - 22:48:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 398 : avg forward pass time 14.39 +[Apr 25, 2026 - 22:48:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 398 : backward pass time 11.54 +[Apr 25, 2026 - 22:48:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 398 : optimizer step time 0.99 +[Apr 25, 2026 - 22:48:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 398-------------------- +[Apr 25, 2026 - 22:48:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0272 iter count: 1.0 +[Apr 25, 2026 - 22:48:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0272 iter count: 1.0 +[Apr 25, 2026 - 22:48:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:48:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:49:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094473 43.160492 43.004936 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.724121 70.738281 70.681641 +util 95.000000 99.000000 90.000000 +[Apr 25, 2026 - 22:49:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 399 : avg iteration time 49.92 seconds +[Apr 25, 2026 - 22:49:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 399 : data loading time 0.00 +[Apr 25, 2026 - 22:49:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 399 : avg forward pass time 12.91 +[Apr 25, 2026 - 22:49:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 399 : backward pass time 11.56 +[Apr 25, 2026 - 22:49:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 399 : optimizer step time 0.99 +[Apr 25, 2026 - 22:49:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 399-------------------- +[Apr 25, 2026 - 22:49:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0285 iter count: 1.0 +[Apr 25, 2026 - 22:49:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0285 iter count: 1.0 +[Apr 25, 2026 - 22:49:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:49:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094198 43.159405 43.004921 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.537109 135.539062 135.535156 +util 70.000000 74.000000 63.000000 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 400 : avg iteration time 167.14 seconds +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 400 : data loading time 0.00 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 400 : avg forward pass time 74.30 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 400 : backward pass time 9.14 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 400 : optimizer step time 0.35 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 400-------------------- +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 845455.9219 iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5570 iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 845455.9219iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 5.2254iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.0775 iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.3171 iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 1352609.5000iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 63.7582iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 16753489.0000iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5570iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.4719 iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 7659.5640iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.5225 iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 8475.5342iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 6.5879 iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 527458.8125iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.3902 iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 355156.0312iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0278 iter count: 1.0 +[Apr 25, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 25, 2026 - 22:52:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 15.65 seconds +[Apr 25, 2026 - 22:52:30 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 400 +[Apr 25, 2026 - 22:52:30 | INFO | fastgen.utils.checkpointer:save:301 ] Saving FSDP model to prefix /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000400 +[Apr 25, 2026 - 22:52:47 | SUCCESS | fastgen.utils.checkpointer:save:329 ] Model saved at iteration 400 +[Apr 25, 2026 - 22:52:47 | INFO | fastgen.trainer:validate:358 ] Validation iteration 400 +[Apr 25, 2026 - 22:52:47 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 22:52:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 2.3s +[Apr 25, 2026 - 22:52:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 22:52:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 22:52:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 22:53:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 22:53:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 22:53:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 22:53:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 22:53:11 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 22:53:14 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 22:53:16 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 22:53:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 22:53:21 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 22:53:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 22:53:27 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 22:53:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 22:53:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 22:53:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 25, 2026 - 22:53:37 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 25, 2026 - 22:53:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging val0 stats at iteration 400-------------------- +[Apr 25, 2026 - 22:53:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 25, 2026 - 22:53:57 | INFO | fastgen.callbacks.wandb:on_validation_end:683 ] Logged 10 val videos at iteration 400 +[Apr 25, 2026 - 22:54:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094476 43.162186 43.004375 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.719238 70.837891 70.679688 +util 95.000000 99.000000 91.000000 +[Apr 25, 2026 - 22:54:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 401 : avg iteration time 151.50 seconds +[Apr 25, 2026 - 22:54:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 401 : data loading time 0.00 +[Apr 25, 2026 - 22:54:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 401 : avg forward pass time 12.55 +[Apr 25, 2026 - 22:54:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 401 : backward pass time 11.55 +[Apr 25, 2026 - 22:54:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 401 : optimizer step time 0.98 +[Apr 25, 2026 - 22:54:46 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 401-------------------- +[Apr 25, 2026 - 22:54:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0255 iter count: 1.0 +[Apr 25, 2026 - 22:54:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0255 iter count: 1.0 +[Apr 25, 2026 - 22:54:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:54:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:55:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094470 43.162163 43.004375 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.812012 70.974609 70.757812 +util 97.000000 100.000000 92.000000 +[Apr 25, 2026 - 22:55:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 402 : avg iteration time 47.14 seconds +[Apr 25, 2026 - 22:55:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 402 : data loading time 0.00 +[Apr 25, 2026 - 22:55:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 402 : avg forward pass time 11.51 +[Apr 25, 2026 - 22:55:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 402 : backward pass time 11.56 +[Apr 25, 2026 - 22:55:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 402 : optimizer step time 0.99 +[Apr 25, 2026 - 22:55:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 402-------------------- +[Apr 25, 2026 - 22:55:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0286 iter count: 1.0 +[Apr 25, 2026 - 22:55:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0286 iter count: 1.0 +[Apr 25, 2026 - 22:55:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:55:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:56:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094473 43.162163 43.004387 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.885254 71.033203 70.835938 +util 97.250000 99.000000 92.000000 +[Apr 25, 2026 - 22:56:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 403 : avg iteration time 50.01 seconds +[Apr 25, 2026 - 22:56:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 403 : data loading time 0.00 +[Apr 25, 2026 - 22:56:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 403 : avg forward pass time 12.96 +[Apr 25, 2026 - 22:56:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 403 : backward pass time 11.55 +[Apr 25, 2026 - 22:56:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 403 : optimizer step time 0.97 +[Apr 25, 2026 - 22:56:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 403-------------------- +[Apr 25, 2026 - 22:56:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0316 iter count: 1.0 +[Apr 25, 2026 - 22:56:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0316 iter count: 1.0 +[Apr 25, 2026 - 22:56:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:56:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 22:57:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093878 43.159794 43.004375 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.851074 70.955078 70.816406 +util 97.250000 99.000000 92.000000 +[Apr 25, 2026 - 22:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 404 : avg iteration time 50.01 seconds +[Apr 25, 2026 - 22:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 404 : data loading time 0.00 +[Apr 25, 2026 - 22:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 404 : avg forward pass time 12.97 +[Apr 25, 2026 - 22:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 404 : backward pass time 11.54 +[Apr 25, 2026 - 22:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 404 : optimizer step time 0.97 +[Apr 25, 2026 - 22:57:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 404-------------------- +[Apr 25, 2026 - 22:57:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0246 iter count: 1.0 +[Apr 25, 2026 - 22:57:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0246 iter count: 1.0 +[Apr 25, 2026 - 22:57:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 22:57:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094162 43.159557 43.004826 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.508301 135.539062 135.498047 +util 68.750000 73.000000 63.000000 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 405 : avg iteration time 176.32 seconds +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 405 : data loading time 0.00 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 405 : avg forward pass time 78.88 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 405 : backward pass time 9.15 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 405 : optimizer step time 0.34 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 405-------------------- +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 65148.9116 iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5328 iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 65148.9116iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3209iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.8107 iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.5531 iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 160052.8906iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 37.3883iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 3635105.7500iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5328iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.2791 iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 95.4060 iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.3218 iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 309755.4688iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.1162 iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 3761.2039iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.3002 iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 5434.0762iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0281 iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:00:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 23:01:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094151 43.159534 43.004829 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.456543 73.787109 70.679688 +util 95.500000 99.000000 91.000000 +[Apr 25, 2026 - 23:01:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 406 : avg iteration time 51.49 seconds +[Apr 25, 2026 - 23:01:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 406 : data loading time 0.00 +[Apr 25, 2026 - 23:01:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 406 : avg forward pass time 13.71 +[Apr 25, 2026 - 23:01:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 406 : backward pass time 11.56 +[Apr 25, 2026 - 23:01:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 406 : optimizer step time 0.97 +[Apr 25, 2026 - 23:01:01 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 406-------------------- +[Apr 25, 2026 - 23:01:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0310 iter count: 1.0 +[Apr 25, 2026 - 23:01:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0310 iter count: 1.0 +[Apr 25, 2026 - 23:01:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:01:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:01:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094324 43.160229 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.709473 70.857422 70.660156 +util 98.000000 100.000000 94.000000 +[Apr 25, 2026 - 23:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 407 : avg iteration time 50.13 seconds +[Apr 25, 2026 - 23:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 407 : data loading time 0.00 +[Apr 25, 2026 - 23:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 407 : avg forward pass time 13.02 +[Apr 25, 2026 - 23:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 407 : backward pass time 11.55 +[Apr 25, 2026 - 23:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 407 : optimizer step time 0.97 +[Apr 25, 2026 - 23:01:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 407-------------------- +[Apr 25, 2026 - 23:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0293 iter count: 1.0 +[Apr 25, 2026 - 23:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0293 iter count: 1.0 +[Apr 25, 2026 - 23:01:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:01:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:02:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094383 43.160465 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.476074 73.806641 70.699219 +util 98.250000 100.000000 96.000000 +[Apr 25, 2026 - 23:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 408 : avg iteration time 47.15 seconds +[Apr 25, 2026 - 23:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 408 : data loading time 0.00 +[Apr 25, 2026 - 23:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 408 : avg forward pass time 11.53 +[Apr 25, 2026 - 23:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 408 : backward pass time 11.55 +[Apr 25, 2026 - 23:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 408 : optimizer step time 0.97 +[Apr 25, 2026 - 23:02:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 408-------------------- +[Apr 25, 2026 - 23:02:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0310 iter count: 1.0 +[Apr 25, 2026 - 23:02:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0310 iter count: 1.0 +[Apr 25, 2026 - 23:02:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:02:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:03:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094383 43.160465 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.563965 73.806641 70.816406 +util 95.500000 99.000000 91.000000 +[Apr 25, 2026 - 23:03:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 409 : avg iteration time 50.12 seconds +[Apr 25, 2026 - 23:03:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 409 : data loading time 0.00 +[Apr 25, 2026 - 23:03:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 409 : avg forward pass time 13.02 +[Apr 25, 2026 - 23:03:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 409 : backward pass time 11.53 +[Apr 25, 2026 - 23:03:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 409 : optimizer step time 0.98 +[Apr 25, 2026 - 23:03:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 409-------------------- +[Apr 25, 2026 - 23:03:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0288 iter count: 1.0 +[Apr 25, 2026 - 23:03:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0288 iter count: 1.0 +[Apr 25, 2026 - 23:03:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:03:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094159 43.159569 43.004826 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.488770 135.519531 135.478516 +util 70.000000 77.000000 65.000000 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 410 : avg iteration time 173.28 seconds +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 410 : data loading time 0.00 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 410 : avg forward pass time 77.35 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 410 : backward pass time 9.17 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 410 : optimizer step time 0.34 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 410-------------------- +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 66653.2026 iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4171 iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 66653.2026iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3016iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6057 iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.5367 iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 187876.0000iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.3582 iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 3518131.5000iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4171iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.5732 iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1269.6177iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.9230 iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2555.7422iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.9486 iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 19873.4238iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.1839 iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 4306.4385iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0290 iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:06:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 23:07:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094166 43.159595 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.621582 70.660156 70.505859 +util 98.250000 100.000000 95.000000 +[Apr 25, 2026 - 23:07:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 411 : avg iteration time 54.26 seconds +[Apr 25, 2026 - 23:07:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 411 : data loading time 0.00 +[Apr 25, 2026 - 23:07:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 411 : avg forward pass time 15.09 +[Apr 25, 2026 - 23:07:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 411 : backward pass time 11.54 +[Apr 25, 2026 - 23:07:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 411 : optimizer step time 0.97 +[Apr 25, 2026 - 23:07:16 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 411-------------------- +[Apr 25, 2026 - 23:07:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0266 iter count: 1.0 +[Apr 25, 2026 - 23:07:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0266 iter count: 1.0 +[Apr 25, 2026 - 23:07:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:07:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:08:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094159 43.159569 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.704590 70.757812 70.544922 +util 97.750000 100.000000 93.000000 +[Apr 25, 2026 - 23:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 412 : avg iteration time 49.98 seconds +[Apr 25, 2026 - 23:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 412 : data loading time 0.00 +[Apr 25, 2026 - 23:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 412 : avg forward pass time 12.94 +[Apr 25, 2026 - 23:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 412 : backward pass time 11.56 +[Apr 25, 2026 - 23:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 412 : optimizer step time 0.99 +[Apr 25, 2026 - 23:08:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 412-------------------- +[Apr 25, 2026 - 23:08:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0265 iter count: 1.0 +[Apr 25, 2026 - 23:08:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0265 iter count: 1.0 +[Apr 25, 2026 - 23:08:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:08:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:08:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094160 43.159569 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.645996 70.679688 70.544922 +util 98.250000 100.000000 94.000000 +[Apr 25, 2026 - 23:08:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 413 : avg iteration time 50.01 seconds +[Apr 25, 2026 - 23:08:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 413 : data loading time 0.00 +[Apr 25, 2026 - 23:08:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 413 : avg forward pass time 12.98 +[Apr 25, 2026 - 23:08:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 413 : backward pass time 11.53 +[Apr 25, 2026 - 23:08:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 413 : optimizer step time 0.99 +[Apr 25, 2026 - 23:08:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 413-------------------- +[Apr 25, 2026 - 23:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0274 iter count: 1.0 +[Apr 25, 2026 - 23:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0274 iter count: 1.0 +[Apr 25, 2026 - 23:08:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:08:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:09:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094330 43.160240 43.004837 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.685059 70.994141 70.582031 +util 98.000000 100.000000 95.000000 +[Apr 25, 2026 - 23:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 414 : avg iteration time 49.94 seconds +[Apr 25, 2026 - 23:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 414 : data loading time 0.00 +[Apr 25, 2026 - 23:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 414 : avg forward pass time 12.93 +[Apr 25, 2026 - 23:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 414 : backward pass time 11.56 +[Apr 25, 2026 - 23:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 414 : optimizer step time 0.98 +[Apr 25, 2026 - 23:09:46 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 414-------------------- +[Apr 25, 2026 - 23:09:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0286 iter count: 1.0 +[Apr 25, 2026 - 23:09:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0286 iter count: 1.0 +[Apr 25, 2026 - 23:09:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:09:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094161 43.159561 43.004826 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.473633 135.519531 135.457031 +util 69.250000 73.000000 63.000000 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 415 : avg iteration time 170.05 seconds +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 415 : data loading time 0.00 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 415 : avg forward pass time 75.74 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 415 : backward pass time 9.15 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 415 : optimizer step time 0.34 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 415-------------------- +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 1111940.9836 iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5259 iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 1111940.9836iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.7040iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.8233 iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.7790 iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 1418312.6250iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 38.3422iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 42203508.0000iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5259iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.4795 iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 7776.9648iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.4131 iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 50324.3516iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.5294 iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 8593.9932iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 1.8303 iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 38.8883 iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0269 iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:12:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 23:13:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094210 43.159771 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.699707 70.757812 70.525391 +util 96.000000 99.000000 92.000000 +[Apr 25, 2026 - 23:13:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 416 : avg iteration time 51.29 seconds +[Apr 25, 2026 - 23:13:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 416 : data loading time 0.00 +[Apr 25, 2026 - 23:13:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 416 : avg forward pass time 13.63 +[Apr 25, 2026 - 23:13:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 416 : backward pass time 11.53 +[Apr 25, 2026 - 23:13:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 416 : optimizer step time 0.98 +[Apr 25, 2026 - 23:13:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 416-------------------- +[Apr 25, 2026 - 23:13:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0311 iter count: 1.0 +[Apr 25, 2026 - 23:13:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0311 iter count: 1.0 +[Apr 25, 2026 - 23:13:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:13:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:14:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094212 43.159771 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.709473 70.757812 70.564453 +util 97.250000 100.000000 92.000000 +[Apr 25, 2026 - 23:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 417 : avg iteration time 47.10 seconds +[Apr 25, 2026 - 23:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 417 : data loading time 0.00 +[Apr 25, 2026 - 23:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 417 : avg forward pass time 11.52 +[Apr 25, 2026 - 23:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 417 : backward pass time 11.51 +[Apr 25, 2026 - 23:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 417 : optimizer step time 0.97 +[Apr 25, 2026 - 23:14:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 417-------------------- +[Apr 25, 2026 - 23:14:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0278 iter count: 1.0 +[Apr 25, 2026 - 23:14:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0278 iter count: 1.0 +[Apr 25, 2026 - 23:14:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:14:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:15:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094210 43.159771 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.704590 70.757812 70.544922 +util 96.750000 100.000000 90.000000 +[Apr 25, 2026 - 23:15:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 418 : avg iteration time 52.86 seconds +[Apr 25, 2026 - 23:15:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 418 : data loading time 0.00 +[Apr 25, 2026 - 23:15:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 418 : avg forward pass time 14.40 +[Apr 25, 2026 - 23:15:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 418 : backward pass time 11.54 +[Apr 25, 2026 - 23:15:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 418 : optimizer step time 0.99 +[Apr 25, 2026 - 23:15:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 418-------------------- +[Apr 25, 2026 - 23:15:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0247 iter count: 1.0 +[Apr 25, 2026 - 23:15:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0247 iter count: 1.0 +[Apr 25, 2026 - 23:15:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:15:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:16:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094210 43.159771 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.602051 70.640625 70.486328 +util 95.500000 99.000000 92.000000 +[Apr 25, 2026 - 23:16:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 419 : avg iteration time 53.07 seconds +[Apr 25, 2026 - 23:16:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 419 : data loading time 0.00 +[Apr 25, 2026 - 23:16:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 419 : avg forward pass time 14.47 +[Apr 25, 2026 - 23:16:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 419 : backward pass time 11.59 +[Apr 25, 2026 - 23:16:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 419 : optimizer step time 0.99 +[Apr 25, 2026 - 23:16:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 419-------------------- +[Apr 25, 2026 - 23:16:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0277 iter count: 1.0 +[Apr 25, 2026 - 23:16:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0277 iter count: 1.0 +[Apr 25, 2026 - 23:16:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:16:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094215 43.159771 43.004848 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.502930 135.519531 135.496094 +util 72.000000 79.000000 63.000000 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 420 : avg iteration time 175.76 seconds +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 420 : data loading time 0.00 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 420 : avg forward pass time 78.60 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 420 : backward pass time 9.13 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 420 : optimizer step time 0.34 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 420-------------------- +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 233429.0996 iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6321 iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 233429.0996iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.9884iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.3390 iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.6179 iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 360791.1250iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 107.5541iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 4138510.7500iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6321iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.6584 iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 607308.1250iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.0290 iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 172477.4062iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.1930 iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 32402.0742iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.4994 iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 59803.1523iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0305 iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:18:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 23:19:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094215 43.159775 43.004837 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.689941 70.777344 70.427734 +util 98.000000 100.000000 94.000000 +[Apr 25, 2026 - 23:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 421 : avg iteration time 54.53 seconds +[Apr 25, 2026 - 23:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 421 : data loading time 0.00 +[Apr 25, 2026 - 23:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 421 : avg forward pass time 15.22 +[Apr 25, 2026 - 23:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 421 : backward pass time 11.55 +[Apr 25, 2026 - 23:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 421 : optimizer step time 0.98 +[Apr 25, 2026 - 23:19:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 421-------------------- +[Apr 25, 2026 - 23:19:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0273 iter count: 1.0 +[Apr 25, 2026 - 23:19:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0273 iter count: 1.0 +[Apr 25, 2026 - 23:19:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:19:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:20:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094210 43.159771 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.611816 70.621094 70.583984 +util 98.750000 100.000000 96.000000 +[Apr 25, 2026 - 23:20:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 422 : avg iteration time 47.02 seconds +[Apr 25, 2026 - 23:20:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 422 : data loading time 0.00 +[Apr 25, 2026 - 23:20:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 422 : avg forward pass time 11.48 +[Apr 25, 2026 - 23:20:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 422 : backward pass time 11.54 +[Apr 25, 2026 - 23:20:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 422 : optimizer step time 0.97 +[Apr 25, 2026 - 23:20:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 422-------------------- +[Apr 25, 2026 - 23:20:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0329 iter count: 1.0 +[Apr 25, 2026 - 23:20:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0329 iter count: 1.0 +[Apr 25, 2026 - 23:20:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:20:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:21:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094272 43.160019 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.743652 70.796875 70.583984 +util 96.500000 100.000000 90.000000 +[Apr 25, 2026 - 23:21:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 423 : avg iteration time 49.94 seconds +[Apr 25, 2026 - 23:21:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 423 : data loading time 0.00 +[Apr 25, 2026 - 23:21:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 423 : avg forward pass time 12.96 +[Apr 25, 2026 - 23:21:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 423 : backward pass time 11.52 +[Apr 25, 2026 - 23:21:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 423 : optimizer step time 0.97 +[Apr 25, 2026 - 23:21:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 423-------------------- +[Apr 25, 2026 - 23:21:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0309 iter count: 1.0 +[Apr 25, 2026 - 23:21:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0309 iter count: 1.0 +[Apr 25, 2026 - 23:21:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:21:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:22:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094275 43.160019 43.004829 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.714355 70.757812 70.583984 +util 97.250000 99.000000 92.000000 +[Apr 25, 2026 - 23:22:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 424 : avg iteration time 49.90 seconds +[Apr 25, 2026 - 23:22:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 424 : data loading time 0.00 +[Apr 25, 2026 - 23:22:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 424 : avg forward pass time 12.92 +[Apr 25, 2026 - 23:22:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 424 : backward pass time 11.53 +[Apr 25, 2026 - 23:22:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 424 : optimizer step time 0.98 +[Apr 25, 2026 - 23:22:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 424-------------------- +[Apr 25, 2026 - 23:22:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0272 iter count: 1.0 +[Apr 25, 2026 - 23:22:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0272 iter count: 1.0 +[Apr 25, 2026 - 23:22:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:22:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094146 43.159500 43.004826 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.513184 135.517578 135.500000 +util 67.500000 74.000000 63.000000 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 425 : avg iteration time 166.87 seconds +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 425 : data loading time 0.00 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 425 : avg forward pass time 74.13 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 425 : backward pass time 9.14 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 425 : optimizer step time 0.34 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 425-------------------- +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 47250.4575 iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4248 iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 47250.4575iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3894iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.1357 iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.3313 iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 119020.8203iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 9.6924 iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2332877.2500iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4248iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.7991 iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 14739.0420iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.0649 iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 3393.9163iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.8404 iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 16007.7168iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 7.3313 iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 2332877.2500iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0306 iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:25:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 23:25:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094139 43.159489 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.709473 70.718750 70.681641 +util 98.250000 100.000000 96.000000 +[Apr 25, 2026 - 23:25:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 426 : avg iteration time 54.43 seconds +[Apr 25, 2026 - 23:25:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 426 : data loading time 0.00 +[Apr 25, 2026 - 23:25:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 426 : avg forward pass time 15.19 +[Apr 25, 2026 - 23:25:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 426 : backward pass time 11.52 +[Apr 25, 2026 - 23:25:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 426 : optimizer step time 0.99 +[Apr 25, 2026 - 23:25:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 426-------------------- +[Apr 25, 2026 - 23:25:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0281 iter count: 1.0 +[Apr 25, 2026 - 23:25:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0281 iter count: 1.0 +[Apr 25, 2026 - 23:25:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:25:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:26:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094144 43.159485 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.719238 70.738281 70.662109 +util 98.250000 100.000000 94.000000 +[Apr 25, 2026 - 23:26:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 427 : avg iteration time 52.85 seconds +[Apr 25, 2026 - 23:26:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 427 : data loading time 0.00 +[Apr 25, 2026 - 23:26:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 427 : avg forward pass time 14.41 +[Apr 25, 2026 - 23:26:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 427 : backward pass time 11.53 +[Apr 25, 2026 - 23:26:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 427 : optimizer step time 0.98 +[Apr 25, 2026 - 23:26:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 427-------------------- +[Apr 25, 2026 - 23:26:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0284 iter count: 1.0 +[Apr 25, 2026 - 23:26:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0284 iter count: 1.0 +[Apr 25, 2026 - 23:26:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:26:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:27:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094144 43.159485 43.004848 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.528809 70.662109 70.484375 +util 96.000000 99.000000 92.000000 +[Apr 25, 2026 - 23:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 428 : avg iteration time 50.07 seconds +[Apr 25, 2026 - 23:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 428 : data loading time 0.00 +[Apr 25, 2026 - 23:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 428 : avg forward pass time 13.01 +[Apr 25, 2026 - 23:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 428 : backward pass time 11.54 +[Apr 25, 2026 - 23:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 428 : optimizer step time 0.98 +[Apr 25, 2026 - 23:27:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 428-------------------- +[Apr 25, 2026 - 23:27:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0316 iter count: 1.0 +[Apr 25, 2026 - 23:27:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0316 iter count: 1.0 +[Apr 25, 2026 - 23:27:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:27:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:28:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094138 43.159485 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.655762 70.701172 70.640625 +util 98.000000 99.000000 95.000000 +[Apr 25, 2026 - 23:28:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 429 : avg iteration time 49.93 seconds +[Apr 25, 2026 - 23:28:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 429 : data loading time 0.00 +[Apr 25, 2026 - 23:28:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 429 : avg forward pass time 12.94 +[Apr 25, 2026 - 23:28:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 429 : backward pass time 11.53 +[Apr 25, 2026 - 23:28:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 429 : optimizer step time 0.97 +[Apr 25, 2026 - 23:28:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 429-------------------- +[Apr 25, 2026 - 23:28:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0278 iter count: 1.0 +[Apr 25, 2026 - 23:28:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0278 iter count: 1.0 +[Apr 25, 2026 - 23:28:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:28:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093835 43.158268 43.004826 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.517578 135.537109 135.458984 +util 69.000000 73.000000 63.000000 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 430 : avg iteration time 175.38 seconds +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 430 : data loading time 0.00 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 430 : avg forward pass time 78.41 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 430 : backward pass time 9.16 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 430 : optimizer step time 0.34 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 430-------------------- +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 1148373.2207 iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5350 iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 1148373.2207iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 5.1492iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.2587 iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.6941 iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 1467858.0000iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 12.3952iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 35613492.0000iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5350iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.6817 iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 11653.5605iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.3364 iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 5842.2241iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 6.1623 iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 225169.7344iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 1.2587 iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 12.3952 iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0301 iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:31:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 23:32:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093834 43.158268 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.729004 70.757812 70.642578 +util 94.750000 99.000000 91.000000 +[Apr 25, 2026 - 23:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 431 : avg iteration time 51.31 seconds +[Apr 25, 2026 - 23:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 431 : data loading time 0.00 +[Apr 25, 2026 - 23:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 431 : avg forward pass time 13.62 +[Apr 25, 2026 - 23:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 431 : backward pass time 11.55 +[Apr 25, 2026 - 23:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 431 : optimizer step time 0.99 +[Apr 25, 2026 - 23:32:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 431-------------------- +[Apr 25, 2026 - 23:32:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0321 iter count: 1.0 +[Apr 25, 2026 - 23:32:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0321 iter count: 1.0 +[Apr 25, 2026 - 23:32:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:32:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:33:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094080 43.159252 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.670410 70.699219 70.583984 +util 97.250000 99.000000 92.000000 +[Apr 25, 2026 - 23:33:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 432 : avg iteration time 49.96 seconds +[Apr 25, 2026 - 23:33:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 432 : data loading time 0.00 +[Apr 25, 2026 - 23:33:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 432 : avg forward pass time 12.94 +[Apr 25, 2026 - 23:33:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 432 : backward pass time 11.55 +[Apr 25, 2026 - 23:33:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 432 : optimizer step time 0.98 +[Apr 25, 2026 - 23:33:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 432-------------------- +[Apr 25, 2026 - 23:33:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0341 iter count: 1.0 +[Apr 25, 2026 - 23:33:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0341 iter count: 1.0 +[Apr 25, 2026 - 23:33:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:33:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:33:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094084 43.159252 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.660645 70.679688 70.603516 +util 97.500000 99.000000 93.000000 +[Apr 25, 2026 - 23:33:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 433 : avg iteration time 47.01 seconds +[Apr 25, 2026 - 23:33:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 433 : data loading time 0.00 +[Apr 25, 2026 - 23:33:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 433 : avg forward pass time 11.49 +[Apr 25, 2026 - 23:33:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 433 : backward pass time 11.53 +[Apr 25, 2026 - 23:33:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 433 : optimizer step time 0.98 +[Apr 25, 2026 - 23:33:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 433-------------------- +[Apr 25, 2026 - 23:33:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0323 iter count: 1.0 +[Apr 25, 2026 - 23:33:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0323 iter count: 1.0 +[Apr 25, 2026 - 23:33:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:33:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:34:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094133 43.159447 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.572754 70.603516 70.562500 +util 98.000000 99.000000 96.000000 +[Apr 25, 2026 - 23:34:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 434 : avg iteration time 49.89 seconds +[Apr 25, 2026 - 23:34:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 434 : data loading time 0.00 +[Apr 25, 2026 - 23:34:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 434 : avg forward pass time 12.91 +[Apr 25, 2026 - 23:34:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 434 : backward pass time 11.55 +[Apr 25, 2026 - 23:34:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 434 : optimizer step time 0.97 +[Apr 25, 2026 - 23:34:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 434-------------------- +[Apr 25, 2026 - 23:34:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0317 iter count: 1.0 +[Apr 25, 2026 - 23:34:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0317 iter count: 1.0 +[Apr 25, 2026 - 23:34:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:34:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093672 43.157619 43.004826 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.503418 135.517578 135.460938 +util 65.750000 72.000000 61.000000 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 435 : avg iteration time 172.55 seconds +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 435 : data loading time 0.00 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 435 : avg forward pass time 77.03 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 435 : backward pass time 9.12 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 435 : optimizer step time 0.34 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 435-------------------- +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 860209.1069 iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5526 iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 860209.1069iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.7438iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.7564 iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.5210 iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 1280376.8750iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 33.5432iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 25192024.0000iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5526iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.2736 iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 5152.1958iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.4745 iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 141.0435iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 7.3685 iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 2513099.0000iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.2700 iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 692.3414iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0314 iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:37:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 23:38:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093741 43.157852 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.714355 70.757812 70.583984 +util 96.250000 99.000000 90.000000 +[Apr 25, 2026 - 23:38:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 436 : avg iteration time 51.34 seconds +[Apr 25, 2026 - 23:38:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 436 : data loading time 0.00 +[Apr 25, 2026 - 23:38:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 436 : avg forward pass time 13.63 +[Apr 25, 2026 - 23:38:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 436 : backward pass time 11.54 +[Apr 25, 2026 - 23:38:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 436 : optimizer step time 0.98 +[Apr 25, 2026 - 23:38:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 436-------------------- +[Apr 25, 2026 - 23:38:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0365 iter count: 1.0 +[Apr 25, 2026 - 23:38:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0365 iter count: 1.0 +[Apr 25, 2026 - 23:38:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:38:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:39:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093967 43.158802 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.606934 70.640625 70.505859 +util 97.000000 99.000000 92.000000 +[Apr 25, 2026 - 23:39:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 437 : avg iteration time 52.75 seconds +[Apr 25, 2026 - 23:39:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 437 : data loading time 0.00 +[Apr 25, 2026 - 23:39:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 437 : avg forward pass time 14.35 +[Apr 25, 2026 - 23:39:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 437 : backward pass time 11.54 +[Apr 25, 2026 - 23:39:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 437 : optimizer step time 0.97 +[Apr 25, 2026 - 23:39:22 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 437-------------------- +[Apr 25, 2026 - 23:39:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0305 iter count: 1.0 +[Apr 25, 2026 - 23:39:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0305 iter count: 1.0 +[Apr 25, 2026 - 23:39:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:39:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:40:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093967 43.158802 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.689941 70.699219 70.662109 +util 96.250000 99.000000 93.000000 +[Apr 25, 2026 - 23:40:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 438 : avg iteration time 46.94 seconds +[Apr 25, 2026 - 23:40:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 438 : data loading time 0.00 +[Apr 25, 2026 - 23:40:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 438 : avg forward pass time 11.45 +[Apr 25, 2026 - 23:40:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 438 : backward pass time 11.53 +[Apr 25, 2026 - 23:40:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 438 : optimizer step time 0.97 +[Apr 25, 2026 - 23:40:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 438-------------------- +[Apr 25, 2026 - 23:40:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0363 iter count: 1.0 +[Apr 25, 2026 - 23:40:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0363 iter count: 1.0 +[Apr 25, 2026 - 23:40:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:40:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:40:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094030 43.159027 43.004833 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.602051 70.662109 70.582031 +util 98.500000 100.000000 96.000000 +[Apr 25, 2026 - 23:40:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 439 : avg iteration time 49.91 seconds +[Apr 25, 2026 - 23:40:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 439 : data loading time 0.00 +[Apr 25, 2026 - 23:40:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 439 : avg forward pass time 12.91 +[Apr 25, 2026 - 23:40:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 439 : backward pass time 11.55 +[Apr 25, 2026 - 23:40:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 439 : optimizer step time 0.99 +[Apr 25, 2026 - 23:40:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 439-------------------- +[Apr 25, 2026 - 23:40:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0328 iter count: 1.0 +[Apr 25, 2026 - 23:40:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0328 iter count: 1.0 +[Apr 25, 2026 - 23:40:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:40:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093817 43.158108 43.004871 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.434082 135.478516 135.417969 +util 67.250000 72.000000 61.000000 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 440 : avg iteration time 178.33 seconds +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 440 : data loading time 0.00 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 440 : avg forward pass time 79.91 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 440 : backward pass time 9.12 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 440 : optimizer step time 0.34 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 440-------------------- +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 41752.8005 iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5641 iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 41752.8005iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.5216iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.9898 iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0623 iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 58849.5508iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 7.2391 iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1362056.1250iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5641iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.0446 iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 3258.8835iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.4325 iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 129.6677iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.1238 iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 516.7596iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.0947 iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 3602.6279iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0313 iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:43:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 23:44:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093794 43.158108 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.626465 70.660156 70.525391 +util 97.000000 99.000000 94.000000 +[Apr 25, 2026 - 23:44:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 441 : avg iteration time 48.38 seconds +[Apr 25, 2026 - 23:44:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 441 : data loading time 0.00 +[Apr 25, 2026 - 23:44:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 441 : avg forward pass time 12.17 +[Apr 25, 2026 - 23:44:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 441 : backward pass time 11.52 +[Apr 25, 2026 - 23:44:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 441 : optimizer step time 0.97 +[Apr 25, 2026 - 23:44:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 441-------------------- +[Apr 25, 2026 - 23:44:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0432 iter count: 1.0 +[Apr 25, 2026 - 23:44:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0432 iter count: 1.0 +[Apr 25, 2026 - 23:44:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:44:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:45:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094154 43.159550 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.680176 70.738281 70.505859 +util 95.500000 99.000000 92.000000 +[Apr 25, 2026 - 23:45:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 442 : avg iteration time 49.97 seconds +[Apr 25, 2026 - 23:45:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 442 : data loading time 0.00 +[Apr 25, 2026 - 23:45:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 442 : avg forward pass time 12.96 +[Apr 25, 2026 - 23:45:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 442 : backward pass time 11.54 +[Apr 25, 2026 - 23:45:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 442 : optimizer step time 0.96 +[Apr 25, 2026 - 23:45:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 442-------------------- +[Apr 25, 2026 - 23:45:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0370 iter count: 1.0 +[Apr 25, 2026 - 23:45:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0370 iter count: 1.0 +[Apr 25, 2026 - 23:45:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:45:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:46:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094157 43.159561 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.523926 70.525391 70.523438 +util 98.000000 100.000000 94.000000 +[Apr 25, 2026 - 23:46:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 443 : avg iteration time 47.09 seconds +[Apr 25, 2026 - 23:46:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 443 : data loading time 0.00 +[Apr 25, 2026 - 23:46:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 443 : avg forward pass time 11.50 +[Apr 25, 2026 - 23:46:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 443 : backward pass time 11.55 +[Apr 25, 2026 - 23:46:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 443 : optimizer step time 0.97 +[Apr 25, 2026 - 23:46:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 443-------------------- +[Apr 25, 2026 - 23:46:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0388 iter count: 1.0 +[Apr 25, 2026 - 23:46:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0388 iter count: 1.0 +[Apr 25, 2026 - 23:46:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:46:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:47:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094234 43.159801 43.004871 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.689941 70.738281 70.544922 +util 94.750000 99.000000 91.000000 +[Apr 25, 2026 - 23:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 444 : avg iteration time 49.90 seconds +[Apr 25, 2026 - 23:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 444 : data loading time 0.00 +[Apr 25, 2026 - 23:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 444 : avg forward pass time 12.91 +[Apr 25, 2026 - 23:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 444 : backward pass time 11.54 +[Apr 25, 2026 - 23:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 444 : optimizer step time 0.98 +[Apr 25, 2026 - 23:47:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 444-------------------- +[Apr 25, 2026 - 23:47:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0367 iter count: 1.0 +[Apr 25, 2026 - 23:47:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0367 iter count: 1.0 +[Apr 25, 2026 - 23:47:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:47:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093937 43.158657 43.004826 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.541016 135.554688 135.500000 +util 71.250000 77.000000 63.000000 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 445 : avg iteration time 175.78 seconds +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 445 : data loading time 0.00 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 445 : avg forward pass time 78.60 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 445 : backward pass time 9.14 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 445 : optimizer step time 0.35 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 445-------------------- +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 174220.3629 iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5522 iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 174220.3629iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4109iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.9484 iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.7700 iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 350373.8438iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 6.6647 iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 5610052.0000iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5522iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.5970 iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 180.1872iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.0921 iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 26483.4844iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.4241 iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 942.2085iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.7978 iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 14699.7646iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0363 iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:50:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 23:51:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.093995 43.158909 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.694824 70.718750 70.623047 +util 95.000000 100.000000 91.000000 +[Apr 25, 2026 - 23:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 446 : avg iteration time 54.43 seconds +[Apr 25, 2026 - 23:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 446 : data loading time 0.00 +[Apr 25, 2026 - 23:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 446 : avg forward pass time 15.18 +[Apr 25, 2026 - 23:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 446 : backward pass time 11.55 +[Apr 25, 2026 - 23:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 446 : optimizer step time 0.99 +[Apr 25, 2026 - 23:51:03 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 446-------------------- +[Apr 25, 2026 - 23:51:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0333 iter count: 1.0 +[Apr 25, 2026 - 23:51:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0333 iter count: 1.0 +[Apr 25, 2026 - 23:51:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:51:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:51:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094239 43.159885 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.689941 70.738281 70.544922 +util 95.750000 99.000000 91.000000 +[Apr 25, 2026 - 23:51:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 447 : avg iteration time 49.98 seconds +[Apr 25, 2026 - 23:51:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 447 : data loading time 0.00 +[Apr 25, 2026 - 23:51:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 447 : avg forward pass time 12.96 +[Apr 25, 2026 - 23:51:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 447 : backward pass time 11.53 +[Apr 25, 2026 - 23:51:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 447 : optimizer step time 0.99 +[Apr 25, 2026 - 23:51:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 447-------------------- +[Apr 25, 2026 - 23:51:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0399 iter count: 1.0 +[Apr 25, 2026 - 23:51:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0399 iter count: 1.0 +[Apr 25, 2026 - 23:51:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:51:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:52:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094252 43.159885 43.004848 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.704590 70.757812 70.544922 +util 98.000000 100.000000 94.000000 +[Apr 25, 2026 - 23:52:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 448 : avg iteration time 52.92 seconds +[Apr 25, 2026 - 23:52:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 448 : data loading time 0.00 +[Apr 25, 2026 - 23:52:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 448 : avg forward pass time 14.41 +[Apr 25, 2026 - 23:52:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 448 : backward pass time 11.56 +[Apr 25, 2026 - 23:52:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 448 : optimizer step time 0.97 +[Apr 25, 2026 - 23:52:46 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 448-------------------- +[Apr 25, 2026 - 23:52:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0385 iter count: 1.0 +[Apr 25, 2026 - 23:52:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0385 iter count: 1.0 +[Apr 25, 2026 - 23:52:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:52:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:53:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094263 43.159889 43.004848 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.616699 70.640625 70.544922 +util 96.000000 99.000000 91.000000 +[Apr 25, 2026 - 23:53:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 449 : avg iteration time 49.95 seconds +[Apr 25, 2026 - 23:53:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 449 : data loading time 0.00 +[Apr 25, 2026 - 23:53:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 449 : avg forward pass time 12.95 +[Apr 25, 2026 - 23:53:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 449 : backward pass time 11.50 +[Apr 25, 2026 - 23:53:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 449 : optimizer step time 0.99 +[Apr 25, 2026 - 23:53:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 449-------------------- +[Apr 25, 2026 - 23:53:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0361 iter count: 1.0 +[Apr 25, 2026 - 23:53:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0361 iter count: 1.0 +[Apr 25, 2026 - 23:53:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:53:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094010 43.158970 43.004826 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.493652 135.539062 135.478516 +util 67.500000 74.000000 63.000000 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 450 : avg iteration time 172.97 seconds +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 450 : data loading time 0.00 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 450 : avg forward pass time 77.20 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 450 : backward pass time 9.16 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 450 : optimizer step time 0.34 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 450-------------------- +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 52232.9395 iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5387 iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 52232.9395iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.9009iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.1192 iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.2790 iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 120358.4062iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 9.3780 iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2101339.2500iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5387iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.4006 iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 6642.3682iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.0249 iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 3132.8604iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.6570 iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 81959.3125iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.9141 iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 18548.1777iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0391 iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:56:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 25, 2026 - 23:57:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094020 43.158989 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.797363 70.855469 70.623047 +util 98.500000 100.000000 96.000000 +[Apr 25, 2026 - 23:57:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 451 : avg iteration time 54.31 seconds +[Apr 25, 2026 - 23:57:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 451 : data loading time 0.00 +[Apr 25, 2026 - 23:57:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 451 : avg forward pass time 15.12 +[Apr 25, 2026 - 23:57:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 451 : backward pass time 11.53 +[Apr 25, 2026 - 23:57:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 451 : optimizer step time 0.97 +[Apr 25, 2026 - 23:57:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 451-------------------- +[Apr 25, 2026 - 23:57:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0391 iter count: 1.0 +[Apr 25, 2026 - 23:57:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0391 iter count: 1.0 +[Apr 25, 2026 - 23:57:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:57:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:58:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094016 43.158970 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.719238 70.757812 70.603516 +util 97.250000 100.000000 92.000000 +[Apr 25, 2026 - 23:58:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 452 : avg iteration time 49.95 seconds +[Apr 25, 2026 - 23:58:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 452 : data loading time 0.00 +[Apr 25, 2026 - 23:58:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 452 : avg forward pass time 12.93 +[Apr 25, 2026 - 23:58:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 452 : backward pass time 11.55 +[Apr 25, 2026 - 23:58:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 452 : optimizer step time 0.99 +[Apr 25, 2026 - 23:58:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 452-------------------- +[Apr 25, 2026 - 23:58:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0391 iter count: 1.0 +[Apr 25, 2026 - 23:58:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0391 iter count: 1.0 +[Apr 25, 2026 - 23:58:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:58:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:59:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.094016 43.158970 43.004826 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.660645 70.699219 70.544922 +util 95.750000 100.000000 92.000000 +[Apr 25, 2026 - 23:59:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 453 : avg iteration time 52.94 seconds +[Apr 25, 2026 - 23:59:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 453 : data loading time 0.00 +[Apr 25, 2026 - 23:59:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 453 : avg forward pass time 14.43 +[Apr 25, 2026 - 23:59:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 453 : backward pass time 11.56 +[Apr 25, 2026 - 23:59:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 453 : optimizer step time 0.98 +[Apr 25, 2026 - 23:59:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 453-------------------- +[Apr 25, 2026 - 23:59:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0405 iter count: 1.0 +[Apr 25, 2026 - 23:59:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0405 iter count: 1.0 +[Apr 25, 2026 - 23:59:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:59:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 25, 2026 - 23:59:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312178 43.418819 43.205402 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.650879 70.699219 70.505859 +util 97.000000 99.000000 92.000000 +[Apr 25, 2026 - 23:59:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 454 : avg iteration time 53.27 seconds +[Apr 25, 2026 - 23:59:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 454 : data loading time 0.00 +[Apr 25, 2026 - 23:59:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 454 : avg forward pass time 13.92 +[Apr 25, 2026 - 23:59:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 454 : backward pass time 11.52 +[Apr 25, 2026 - 23:59:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 454 : optimizer step time 0.99 +[Apr 25, 2026 - 23:59:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 454-------------------- +[Apr 25, 2026 - 23:59:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0362 iter count: 1.0 +[Apr 25, 2026 - 23:59:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0362 iter count: 1.0 +[Apr 25, 2026 - 23:59:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 25, 2026 - 23:59:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313042 43.420231 43.205910 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.551270 135.556641 135.537109 +util 72.250000 79.000000 64.000000 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 455 : avg iteration time 170.50 seconds +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 455 : data loading time 0.00 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 455 : avg forward pass time 75.99 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 455 : backward pass time 9.12 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 455 : optimizer step time 0.35 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 455-------------------- +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 391944.0488 iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5230 iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 391944.0488iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5403iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7018 iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.1770 iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 581678.4375iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.0697 iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 12661790.0000iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5230iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.8554 iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 121878.4141iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.2365 iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 4784.1899iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.1887 iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 32126.2266iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 0.8266 iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 5.2234 iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0394 iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:02:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 26, 2026 - 00:03:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313683 43.421158 43.206039 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.675293 70.720703 70.660156 +util 97.750000 100.000000 93.000000 +[Apr 26, 2026 - 00:03:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 456 : avg iteration time 51.71 seconds +[Apr 26, 2026 - 00:03:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 456 : data loading time 0.00 +[Apr 26, 2026 - 00:03:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 456 : avg forward pass time 13.69 +[Apr 26, 2026 - 00:03:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 456 : backward pass time 11.80 +[Apr 26, 2026 - 00:03:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 456 : optimizer step time 0.99 +[Apr 26, 2026 - 00:03:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 456-------------------- +[Apr 26, 2026 - 00:03:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0451 iter count: 1.0 +[Apr 26, 2026 - 00:03:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0451 iter count: 1.0 +[Apr 26, 2026 - 00:03:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:03:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:04:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.314054 43.421635 43.206356 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.514160 70.720703 70.445312 +util 96.000000 99.000000 92.000000 +[Apr 26, 2026 - 00:04:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 457 : avg iteration time 52.96 seconds +[Apr 26, 2026 - 00:04:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 457 : data loading time 0.00 +[Apr 26, 2026 - 00:04:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 457 : avg forward pass time 14.44 +[Apr 26, 2026 - 00:04:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 457 : backward pass time 11.55 +[Apr 26, 2026 - 00:04:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 457 : optimizer step time 0.98 +[Apr 26, 2026 - 00:04:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 457-------------------- +[Apr 26, 2026 - 00:04:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0363 iter count: 1.0 +[Apr 26, 2026 - 00:04:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0363 iter count: 1.0 +[Apr 26, 2026 - 00:04:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:04:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:05:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.314492 43.421925 43.207176 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.514160 70.662109 70.464844 +util 96.750000 99.000000 91.000000 +[Apr 26, 2026 - 00:05:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 458 : avg iteration time 52.89 seconds +[Apr 26, 2026 - 00:05:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 458 : data loading time 0.00 +[Apr 26, 2026 - 00:05:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 458 : avg forward pass time 14.40 +[Apr 26, 2026 - 00:05:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 458 : backward pass time 11.57 +[Apr 26, 2026 - 00:05:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 458 : optimizer step time 0.98 +[Apr 26, 2026 - 00:05:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 458-------------------- +[Apr 26, 2026 - 00:05:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0424 iter count: 1.0 +[Apr 26, 2026 - 00:05:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0424 iter count: 1.0 +[Apr 26, 2026 - 00:05:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:05:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:06:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.314595 43.422020 43.207283 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.655762 70.759766 70.621094 +util 98.500000 100.000000 95.000000 +[Apr 26, 2026 - 00:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 459 : avg iteration time 52.97 seconds +[Apr 26, 2026 - 00:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 459 : data loading time 0.00 +[Apr 26, 2026 - 00:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 459 : avg forward pass time 14.45 +[Apr 26, 2026 - 00:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 459 : backward pass time 11.53 +[Apr 26, 2026 - 00:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 459 : optimizer step time 1.00 +[Apr 26, 2026 - 00:06:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 459-------------------- +[Apr 26, 2026 - 00:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0407 iter count: 1.0 +[Apr 26, 2026 - 00:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0407 iter count: 1.0 +[Apr 26, 2026 - 00:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:06:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.314628 43.421040 43.207287 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.502930 135.519531 135.496094 +util 71.500000 75.000000 66.000000 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 460 : avg iteration time 172.80 seconds +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 460 : data loading time 0.00 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 460 : avg forward pass time 77.14 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 460 : backward pass time 9.15 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 460 : optimizer step time 0.34 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 460-------------------- +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 296270.8730 iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5033 iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 296270.8730iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.7116iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.4439 iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.9506 iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 438445.3125iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.4298 iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 8049919.0000iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5033iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 7.1338 iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1571682.5000iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.0286 iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 3156.1914iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.9762 iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 21003.3730iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.0322 iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 173585.9844iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0391 iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:09:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 26, 2026 - 00:10:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.314951 43.421131 43.208138 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.670410 70.701172 70.660156 +util 97.500000 100.000000 93.000000 +[Apr 26, 2026 - 00:10:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 461 : avg iteration time 48.57 seconds +[Apr 26, 2026 - 00:10:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 461 : data loading time 0.00 +[Apr 26, 2026 - 00:10:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 461 : avg forward pass time 12.26 +[Apr 26, 2026 - 00:10:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 461 : backward pass time 11.53 +[Apr 26, 2026 - 00:10:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 461 : optimizer step time 0.98 +[Apr 26, 2026 - 00:10:01 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 461-------------------- +[Apr 26, 2026 - 00:10:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0509 iter count: 1.0 +[Apr 26, 2026 - 00:10:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0509 iter count: 1.0 +[Apr 26, 2026 - 00:10:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:10:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:10:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.315490 43.421879 43.208229 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.738770 70.757812 70.681641 +util 94.250000 99.000000 90.000000 +[Apr 26, 2026 - 00:10:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 462 : avg iteration time 47.11 seconds +[Apr 26, 2026 - 00:10:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 462 : data loading time 0.00 +[Apr 26, 2026 - 00:10:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 462 : avg forward pass time 11.52 +[Apr 26, 2026 - 00:10:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 462 : backward pass time 11.53 +[Apr 26, 2026 - 00:10:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 462 : optimizer step time 0.98 +[Apr 26, 2026 - 00:10:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 462-------------------- +[Apr 26, 2026 - 00:10:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0449 iter count: 1.0 +[Apr 26, 2026 - 00:10:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0449 iter count: 1.0 +[Apr 26, 2026 - 00:10:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:10:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:11:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.316216 43.423115 43.208229 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.748535 70.757812 70.720703 +util 97.500000 99.000000 95.000000 +[Apr 26, 2026 - 00:11:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 463 : avg iteration time 49.85 seconds +[Apr 26, 2026 - 00:11:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 463 : data loading time 0.00 +[Apr 26, 2026 - 00:11:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 463 : avg forward pass time 12.90 +[Apr 26, 2026 - 00:11:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 463 : backward pass time 11.53 +[Apr 26, 2026 - 00:11:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 463 : optimizer step time 0.97 +[Apr 26, 2026 - 00:11:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 463-------------------- +[Apr 26, 2026 - 00:11:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0449 iter count: 1.0 +[Apr 26, 2026 - 00:11:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0449 iter count: 1.0 +[Apr 26, 2026 - 00:11:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:11:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:12:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.316748 43.423397 43.208248 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.743652 70.757812 70.701172 +util 97.000000 99.000000 94.000000 +[Apr 26, 2026 - 00:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 464 : avg iteration time 50.00 seconds +[Apr 26, 2026 - 00:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 464 : data loading time 0.00 +[Apr 26, 2026 - 00:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 464 : avg forward pass time 12.97 +[Apr 26, 2026 - 00:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 464 : backward pass time 11.55 +[Apr 26, 2026 - 00:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 464 : optimizer step time 0.99 +[Apr 26, 2026 - 00:12:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 464-------------------- +[Apr 26, 2026 - 00:12:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0436 iter count: 1.0 +[Apr 26, 2026 - 00:12:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0436 iter count: 1.0 +[Apr 26, 2026 - 00:12:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:12:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.316365 43.421249 43.208263 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.537109 135.597656 135.515625 +util 69.250000 76.000000 65.000000 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 465 : avg iteration time 172.80 seconds +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 465 : data loading time 0.00 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 465 : avg forward pass time 77.12 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 465 : backward pass time 9.14 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 465 : optimizer step time 0.34 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 465-------------------- +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 27898.2833 iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.3966 iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 27898.2833iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3079iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.6375 iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.8709 iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 75152.7109iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 26.4446iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 928967.3750iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.3966iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 1.6375 iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 26.4446 iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.4217 iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 937.5922iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.1181 iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 27897.2188iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.5946 iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 72346.6406iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0418 iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:15:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 26, 2026 - 00:16:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.316501 43.421593 43.208374 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.675293 70.720703 70.660156 +util 97.250000 100.000000 92.000000 +[Apr 26, 2026 - 00:16:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 466 : avg iteration time 48.51 seconds +[Apr 26, 2026 - 00:16:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 466 : data loading time 0.00 +[Apr 26, 2026 - 00:16:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 466 : avg forward pass time 12.23 +[Apr 26, 2026 - 00:16:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 466 : backward pass time 11.51 +[Apr 26, 2026 - 00:16:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 466 : optimizer step time 0.99 +[Apr 26, 2026 - 00:16:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 466-------------------- +[Apr 26, 2026 - 00:16:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0439 iter count: 1.0 +[Apr 26, 2026 - 00:16:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0439 iter count: 1.0 +[Apr 26, 2026 - 00:16:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:16:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:17:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.316815 43.422577 43.208454 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.621582 70.798828 70.562500 +util 97.250000 100.000000 92.000000 +[Apr 26, 2026 - 00:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 467 : avg iteration time 52.93 seconds +[Apr 26, 2026 - 00:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 467 : data loading time 0.00 +[Apr 26, 2026 - 00:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 467 : avg forward pass time 14.40 +[Apr 26, 2026 - 00:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 467 : backward pass time 11.58 +[Apr 26, 2026 - 00:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 467 : optimizer step time 0.98 +[Apr 26, 2026 - 00:17:03 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 467-------------------- +[Apr 26, 2026 - 00:17:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0406 iter count: 1.0 +[Apr 26, 2026 - 00:17:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0406 iter count: 1.0 +[Apr 26, 2026 - 00:17:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:17:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:17:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.316918 43.422684 43.208549 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.597168 70.818359 70.523438 +util 96.500000 99.000000 93.000000 +[Apr 26, 2026 - 00:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 468 : avg iteration time 52.95 seconds +[Apr 26, 2026 - 00:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 468 : data loading time 0.00 +[Apr 26, 2026 - 00:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 468 : avg forward pass time 14.43 +[Apr 26, 2026 - 00:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 468 : backward pass time 11.55 +[Apr 26, 2026 - 00:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 468 : optimizer step time 0.99 +[Apr 26, 2026 - 00:17:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 468-------------------- +[Apr 26, 2026 - 00:17:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0361 iter count: 1.0 +[Apr 26, 2026 - 00:17:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0361 iter count: 1.0 +[Apr 26, 2026 - 00:17:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:17:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:18:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.317256 43.423813 43.208660 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.602051 70.720703 70.562500 +util 96.250000 99.000000 91.000000 +[Apr 26, 2026 - 00:18:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 469 : avg iteration time 50.23 seconds +[Apr 26, 2026 - 00:18:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 469 : data loading time 0.00 +[Apr 26, 2026 - 00:18:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 469 : avg forward pass time 13.08 +[Apr 26, 2026 - 00:18:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 469 : backward pass time 11.54 +[Apr 26, 2026 - 00:18:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 469 : optimizer step time 0.99 +[Apr 26, 2026 - 00:18:46 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 469-------------------- +[Apr 26, 2026 - 00:18:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0386 iter count: 1.0 +[Apr 26, 2026 - 00:18:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0386 iter count: 1.0 +[Apr 26, 2026 - 00:18:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:18:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.317210 43.422577 43.208683 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.488770 135.519531 135.478516 +util 68.000000 73.000000 62.000000 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 470 : avg iteration time 167.21 seconds +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 470 : data loading time 0.00 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 470 : avg forward pass time 74.34 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 470 : backward pass time 9.12 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 470 : optimizer step time 0.34 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 470-------------------- +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 457712.8994 iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5391 iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 457712.8994iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5337iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.8275 iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.5858 iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 1224732.3750iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 5.2330 iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 28675964.0000iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5391iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 7.5794 iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 3831400.7500iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.8420 iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 16059.1240iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.9358 iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 143138.0469iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 0.9069 iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 6.1337 iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0419 iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:21:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 26, 2026 - 00:22:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.317689 43.423519 43.209629 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.812012 70.894531 70.564453 +util 97.500000 100.000000 95.000000 +[Apr 26, 2026 - 00:22:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 471 : avg iteration time 48.37 seconds +[Apr 26, 2026 - 00:22:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 471 : data loading time 0.00 +[Apr 26, 2026 - 00:22:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 471 : avg forward pass time 12.17 +[Apr 26, 2026 - 00:22:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 471 : backward pass time 11.52 +[Apr 26, 2026 - 00:22:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 471 : optimizer step time 0.99 +[Apr 26, 2026 - 00:22:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 471-------------------- +[Apr 26, 2026 - 00:22:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0415 iter count: 1.0 +[Apr 26, 2026 - 00:22:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0415 iter count: 1.0 +[Apr 26, 2026 - 00:22:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:22:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:23:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318204 43.424553 43.210545 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.729004 70.777344 70.583984 +util 95.250000 99.000000 92.000000 +[Apr 26, 2026 - 00:23:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 472 : avg iteration time 52.77 seconds +[Apr 26, 2026 - 00:23:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 472 : data loading time 0.00 +[Apr 26, 2026 - 00:23:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 472 : avg forward pass time 14.36 +[Apr 26, 2026 - 00:23:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 472 : backward pass time 11.53 +[Apr 26, 2026 - 00:23:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 472 : optimizer step time 0.99 +[Apr 26, 2026 - 00:23:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 472-------------------- +[Apr 26, 2026 - 00:23:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0341 iter count: 1.0 +[Apr 26, 2026 - 00:23:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0341 iter count: 1.0 +[Apr 26, 2026 - 00:23:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:23:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:24:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318349 43.425003 43.210636 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.685059 70.759766 70.660156 +util 96.750000 99.000000 91.000000 +[Apr 26, 2026 - 00:24:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 473 : avg iteration time 49.93 seconds +[Apr 26, 2026 - 00:24:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 473 : data loading time 0.00 +[Apr 26, 2026 - 00:24:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 473 : avg forward pass time 12.93 +[Apr 26, 2026 - 00:24:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 473 : backward pass time 11.53 +[Apr 26, 2026 - 00:24:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 473 : optimizer step time 0.99 +[Apr 26, 2026 - 00:24:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 473-------------------- +[Apr 26, 2026 - 00:24:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0368 iter count: 1.0 +[Apr 26, 2026 - 00:24:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0368 iter count: 1.0 +[Apr 26, 2026 - 00:24:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:24:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:24:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318435 43.424744 43.210831 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.812012 70.875000 70.623047 +util 97.500000 99.000000 93.000000 +[Apr 26, 2026 - 00:24:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 474 : avg iteration time 52.82 seconds +[Apr 26, 2026 - 00:24:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 474 : data loading time 0.00 +[Apr 26, 2026 - 00:24:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 474 : avg forward pass time 14.38 +[Apr 26, 2026 - 00:24:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 474 : backward pass time 11.54 +[Apr 26, 2026 - 00:24:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 474 : optimizer step time 0.98 +[Apr 26, 2026 - 00:24:57 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 474-------------------- +[Apr 26, 2026 - 00:24:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0361 iter count: 1.0 +[Apr 26, 2026 - 00:24:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0361 iter count: 1.0 +[Apr 26, 2026 - 00:24:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:24:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318300 43.424004 43.210819 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.444336 135.519531 135.417969 +util 67.250000 72.000000 61.000000 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 475 : avg iteration time 166.90 seconds +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 475 : data loading time 0.00 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 475 : avg forward pass time 74.19 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 475 : backward pass time 9.14 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 475 : optimizer step time 0.34 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 475-------------------- +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 87696.9653 iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4818 iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 87696.9653iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3123iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.4678 iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.4958 iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 203425.6875iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.5485 iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 3241354.2500iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4818iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 7.0847 iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1424470.2500iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.2531 iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 669.3082iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.8597 iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 2251.8196iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.0505 iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 60.4034 iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0381 iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 26, 2026 - 00:28:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318367 43.424038 43.210999 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.689941 70.718750 70.603516 +util 98.500000 100.000000 96.000000 +[Apr 26, 2026 - 00:28:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 476 : avg iteration time 51.26 seconds +[Apr 26, 2026 - 00:28:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 476 : data loading time 0.00 +[Apr 26, 2026 - 00:28:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 476 : avg forward pass time 13.60 +[Apr 26, 2026 - 00:28:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 476 : backward pass time 11.54 +[Apr 26, 2026 - 00:28:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 476 : optimizer step time 0.98 +[Apr 26, 2026 - 00:28:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 476-------------------- +[Apr 26, 2026 - 00:28:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0349 iter count: 1.0 +[Apr 26, 2026 - 00:28:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0349 iter count: 1.0 +[Apr 26, 2026 - 00:28:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:28:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:29:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318718 43.425251 43.211094 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.655762 70.679688 70.583984 +util 97.500000 99.000000 93.000000 +[Apr 26, 2026 - 00:29:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 477 : avg iteration time 49.84 seconds +[Apr 26, 2026 - 00:29:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 477 : data loading time 0.00 +[Apr 26, 2026 - 00:29:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 477 : avg forward pass time 12.91 +[Apr 26, 2026 - 00:29:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 477 : backward pass time 11.52 +[Apr 26, 2026 - 00:29:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 477 : optimizer step time 0.98 +[Apr 26, 2026 - 00:29:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 477-------------------- +[Apr 26, 2026 - 00:29:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0343 iter count: 1.0 +[Apr 26, 2026 - 00:29:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0343 iter count: 1.0 +[Apr 26, 2026 - 00:29:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:29:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:30:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318810 43.425388 43.211094 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.611816 70.621094 70.583984 +util 98.500000 100.000000 96.000000 +[Apr 26, 2026 - 00:30:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 478 : avg iteration time 49.96 seconds +[Apr 26, 2026 - 00:30:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 478 : data loading time 0.00 +[Apr 26, 2026 - 00:30:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 478 : avg forward pass time 12.96 +[Apr 26, 2026 - 00:30:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 478 : backward pass time 11.54 +[Apr 26, 2026 - 00:30:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 478 : optimizer step time 0.98 +[Apr 26, 2026 - 00:30:15 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 478-------------------- +[Apr 26, 2026 - 00:30:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0356 iter count: 1.0 +[Apr 26, 2026 - 00:30:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0356 iter count: 1.0 +[Apr 26, 2026 - 00:30:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:30:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:31:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318943 43.425709 43.211105 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.724121 70.757812 70.623047 +util 97.500000 99.000000 93.000000 +[Apr 26, 2026 - 00:31:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 479 : avg iteration time 52.84 seconds +[Apr 26, 2026 - 00:31:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 479 : data loading time 0.00 +[Apr 26, 2026 - 00:31:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 479 : avg forward pass time 14.41 +[Apr 26, 2026 - 00:31:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 479 : backward pass time 11.52 +[Apr 26, 2026 - 00:31:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 479 : optimizer step time 0.99 +[Apr 26, 2026 - 00:31:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 479-------------------- +[Apr 26, 2026 - 00:31:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0317 iter count: 1.0 +[Apr 26, 2026 - 00:31:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0317 iter count: 1.0 +[Apr 26, 2026 - 00:31:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:31:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318686 43.424519 43.211109 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.552246 135.556641 135.539062 +util 65.500000 73.000000 60.000000 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 480 : avg iteration time 172.64 seconds +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 480 : data loading time 0.00 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 480 : avg forward pass time 77.02 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 480 : backward pass time 9.19 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 480 : optimizer step time 0.34 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 480-------------------- +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 101462.0162 iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4852 iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 101462.0162iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5184iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.8407 iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.8725 iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 291458.9062iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 5.3728 iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 6885743.5000iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4852iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.0139 iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 3065.2473iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.0435 iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 24028.2305iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.6360 iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 78590.1172iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.2948 iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 293518.7812iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0345 iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:34:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 26, 2026 - 00:34:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318685 43.424496 43.211128 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.714355 70.777344 70.525391 +util 97.250000 99.000000 95.000000 +[Apr 26, 2026 - 00:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 481 : avg iteration time 51.44 seconds +[Apr 26, 2026 - 00:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 481 : data loading time 0.00 +[Apr 26, 2026 - 00:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 481 : avg forward pass time 13.69 +[Apr 26, 2026 - 00:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 481 : backward pass time 11.54 +[Apr 26, 2026 - 00:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 481 : optimizer step time 0.99 +[Apr 26, 2026 - 00:34:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 481-------------------- +[Apr 26, 2026 - 00:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0317 iter count: 1.0 +[Apr 26, 2026 - 00:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0317 iter count: 1.0 +[Apr 26, 2026 - 00:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:34:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:35:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318720 43.424541 43.211113 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.694824 70.740234 70.679688 +util 97.500000 100.000000 93.000000 +[Apr 26, 2026 - 00:35:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 482 : avg iteration time 52.84 seconds +[Apr 26, 2026 - 00:35:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 482 : data loading time 0.00 +[Apr 26, 2026 - 00:35:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 482 : avg forward pass time 14.39 +[Apr 26, 2026 - 00:35:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 482 : backward pass time 11.55 +[Apr 26, 2026 - 00:35:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 482 : optimizer step time 0.98 +[Apr 26, 2026 - 00:35:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 482-------------------- +[Apr 26, 2026 - 00:35:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0281 iter count: 1.0 +[Apr 26, 2026 - 00:35:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0281 iter count: 1.0 +[Apr 26, 2026 - 00:35:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:35:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:36:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318761 43.424568 43.211155 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.763184 70.777344 70.720703 +util 97.500000 99.000000 93.000000 +[Apr 26, 2026 - 00:36:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 483 : avg iteration time 52.88 seconds +[Apr 26, 2026 - 00:36:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 483 : data loading time 0.00 +[Apr 26, 2026 - 00:36:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 483 : avg forward pass time 14.41 +[Apr 26, 2026 - 00:36:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 483 : backward pass time 11.56 +[Apr 26, 2026 - 00:36:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 483 : optimizer step time 0.98 +[Apr 26, 2026 - 00:36:37 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 483-------------------- +[Apr 26, 2026 - 00:36:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0294 iter count: 1.0 +[Apr 26, 2026 - 00:36:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0294 iter count: 1.0 +[Apr 26, 2026 - 00:36:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:36:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:37:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318951 43.425255 43.211231 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.753418 70.796875 70.623047 +util 98.250000 99.000000 96.000000 +[Apr 26, 2026 - 00:37:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 484 : avg iteration time 46.96 seconds +[Apr 26, 2026 - 00:37:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 484 : data loading time 0.00 +[Apr 26, 2026 - 00:37:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 484 : avg forward pass time 11.44 +[Apr 26, 2026 - 00:37:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 484 : backward pass time 11.55 +[Apr 26, 2026 - 00:37:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 484 : optimizer step time 0.97 +[Apr 26, 2026 - 00:37:24 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 484-------------------- +[Apr 26, 2026 - 00:37:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0323 iter count: 1.0 +[Apr 26, 2026 - 00:37:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0323 iter count: 1.0 +[Apr 26, 2026 - 00:37:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:37:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318820 43.424614 43.211330 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.488770 135.519531 135.478516 +util 71.250000 76.000000 65.000000 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 485 : avg iteration time 172.99 seconds +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 485 : data loading time 0.00 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 485 : avg forward pass time 77.22 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 485 : backward pass time 9.13 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 485 : optimizer step time 0.34 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 485-------------------- +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 80892.2061 iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5837 iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 80892.2061iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.2055iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6903 iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.2045 iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 105431.8750iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.9774 iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1810433.1250iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5837iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 1.0160 iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 7.6300 iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.2740 iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 38103.0625iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.7407 iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 96896.9531iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.1575 iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 30180.9492iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0309 iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:40:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 26, 2026 - 00:41:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318839 43.424591 43.211308 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.680176 70.738281 70.505859 +util 97.500000 100.000000 93.000000 +[Apr 26, 2026 - 00:41:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 486 : avg iteration time 48.37 seconds +[Apr 26, 2026 - 00:41:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 486 : data loading time 0.00 +[Apr 26, 2026 - 00:41:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 486 : avg forward pass time 12.15 +[Apr 26, 2026 - 00:41:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 486 : backward pass time 11.54 +[Apr 26, 2026 - 00:41:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 486 : optimizer step time 0.97 +[Apr 26, 2026 - 00:41:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 486-------------------- +[Apr 26, 2026 - 00:41:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0259 iter count: 1.0 +[Apr 26, 2026 - 00:41:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0259 iter count: 1.0 +[Apr 26, 2026 - 00:41:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:41:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:41:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318861 43.424614 43.211353 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.631348 70.720703 70.601562 +util 98.500000 100.000000 95.000000 +[Apr 26, 2026 - 00:41:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 487 : avg iteration time 52.83 seconds +[Apr 26, 2026 - 00:41:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 487 : data loading time 0.00 +[Apr 26, 2026 - 00:41:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 487 : avg forward pass time 14.40 +[Apr 26, 2026 - 00:41:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 487 : backward pass time 11.53 +[Apr 26, 2026 - 00:41:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 487 : optimizer step time 0.98 +[Apr 26, 2026 - 00:41:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 487-------------------- +[Apr 26, 2026 - 00:41:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0262 iter count: 1.0 +[Apr 26, 2026 - 00:41:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0262 iter count: 1.0 +[Apr 26, 2026 - 00:41:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:41:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:42:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318926 43.424862 43.211357 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.602051 70.720703 70.562500 +util 94.750000 99.000000 91.000000 +[Apr 26, 2026 - 00:42:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 488 : avg iteration time 49.89 seconds +[Apr 26, 2026 - 00:42:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 488 : data loading time 0.00 +[Apr 26, 2026 - 00:42:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 488 : avg forward pass time 12.92 +[Apr 26, 2026 - 00:42:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 488 : backward pass time 11.54 +[Apr 26, 2026 - 00:42:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 488 : optimizer step time 0.97 +[Apr 26, 2026 - 00:42:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 488-------------------- +[Apr 26, 2026 - 00:42:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0254 iter count: 1.0 +[Apr 26, 2026 - 00:42:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0254 iter count: 1.0 +[Apr 26, 2026 - 00:42:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:42:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:43:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318996 43.425140 43.211338 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.743652 70.757812 70.701172 +util 96.500000 99.000000 93.000000 +[Apr 26, 2026 - 00:43:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 489 : avg iteration time 49.83 seconds +[Apr 26, 2026 - 00:43:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 489 : data loading time 0.00 +[Apr 26, 2026 - 00:43:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 489 : avg forward pass time 12.91 +[Apr 26, 2026 - 00:43:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 489 : backward pass time 11.51 +[Apr 26, 2026 - 00:43:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 489 : optimizer step time 0.97 +[Apr 26, 2026 - 00:43:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 489-------------------- +[Apr 26, 2026 - 00:43:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0280 iter count: 1.0 +[Apr 26, 2026 - 00:43:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0280 iter count: 1.0 +[Apr 26, 2026 - 00:43:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:43:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318852 43.424541 43.211338 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.488770 135.519531 135.478516 +util 72.500000 78.000000 64.000000 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 490 : avg iteration time 175.36 seconds +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 490 : data loading time 0.00 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 490 : avg forward pass time 78.39 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 490 : backward pass time 9.19 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 490 : optimizer step time 0.34 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 490-------------------- +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 191690.3804 iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5066 iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 191690.3804iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5422iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.9825 iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.1120 iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 431239.9062iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 7.1354 iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 11117611.0000iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5066iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.8537 iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 897615.9375iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.9185 iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 18714.0957iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.4399 iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 53094.1406iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.4064 iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 49649.8086iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0272 iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:46:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 26, 2026 - 00:47:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318915 43.424622 43.211426 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.641113 70.818359 70.582031 +util 98.000000 100.000000 96.000000 +[Apr 26, 2026 - 00:47:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 491 : avg iteration time 51.26 seconds +[Apr 26, 2026 - 00:47:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 491 : data loading time 0.00 +[Apr 26, 2026 - 00:47:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 491 : avg forward pass time 13.60 +[Apr 26, 2026 - 00:47:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 491 : backward pass time 11.53 +[Apr 26, 2026 - 00:47:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 491 : optimizer step time 1.00 +[Apr 26, 2026 - 00:47:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 491-------------------- +[Apr 26, 2026 - 00:47:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0288 iter count: 1.0 +[Apr 26, 2026 - 00:47:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0288 iter count: 1.0 +[Apr 26, 2026 - 00:47:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:47:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:48:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318980 43.424622 43.211685 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.611816 70.818359 70.542969 +util 98.500000 100.000000 96.000000 +[Apr 26, 2026 - 00:48:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 492 : avg iteration time 49.87 seconds +[Apr 26, 2026 - 00:48:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 492 : data loading time 0.00 +[Apr 26, 2026 - 00:48:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 492 : avg forward pass time 12.89 +[Apr 26, 2026 - 00:48:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 492 : backward pass time 11.54 +[Apr 26, 2026 - 00:48:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 492 : optimizer step time 0.98 +[Apr 26, 2026 - 00:48:15 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 492-------------------- +[Apr 26, 2026 - 00:48:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0271 iter count: 1.0 +[Apr 26, 2026 - 00:48:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0271 iter count: 1.0 +[Apr 26, 2026 - 00:48:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:48:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:49:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319056 43.424660 43.211685 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.597168 70.818359 70.523438 +util 96.750000 99.000000 94.000000 +[Apr 26, 2026 - 00:49:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 493 : avg iteration time 49.91 seconds +[Apr 26, 2026 - 00:49:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 493 : data loading time 0.00 +[Apr 26, 2026 - 00:49:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 493 : avg forward pass time 12.91 +[Apr 26, 2026 - 00:49:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 493 : backward pass time 11.54 +[Apr 26, 2026 - 00:49:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 493 : optimizer step time 0.99 +[Apr 26, 2026 - 00:49:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 493-------------------- +[Apr 26, 2026 - 00:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0329 iter count: 1.0 +[Apr 26, 2026 - 00:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0329 iter count: 1.0 +[Apr 26, 2026 - 00:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:49:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:49:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319075 43.424644 43.211792 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.582520 70.818359 70.503906 +util 97.750000 99.000000 94.000000 +[Apr 26, 2026 - 00:49:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 494 : avg iteration time 49.95 seconds +[Apr 26, 2026 - 00:49:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 494 : data loading time 0.00 +[Apr 26, 2026 - 00:49:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 494 : avg forward pass time 12.92 +[Apr 26, 2026 - 00:49:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 494 : backward pass time 11.54 +[Apr 26, 2026 - 00:49:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 494 : optimizer step time 0.99 +[Apr 26, 2026 - 00:49:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 494-------------------- +[Apr 26, 2026 - 00:49:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0279 iter count: 1.0 +[Apr 26, 2026 - 00:49:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0279 iter count: 1.0 +[Apr 26, 2026 - 00:49:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:49:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319066 43.424622 43.211796 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.532715 135.556641 135.460938 +util 70.250000 77.000000 66.000000 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 495 : avg iteration time 172.27 seconds +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 495 : data loading time 0.00 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 495 : avg forward pass time 76.87 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 495 : backward pass time 9.14 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 495 : optimizer step time 0.34 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 495-------------------- +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 515585.0688 iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5873 iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 515585.0688iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.6040iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.3838 iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.3060 iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 634606.1875iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 117.6479iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 16387572.0000iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5873iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.5122 iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 8303.1260iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.6191 iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 10283.4863iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.6232 iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 76603.5625iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.7741 iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1897.2242iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0274 iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:52:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 26, 2026 - 00:53:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319202 43.425106 43.211796 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.729004 70.876953 70.679688 +util 97.750000 99.000000 94.000000 +[Apr 26, 2026 - 00:53:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 496 : avg iteration time 51.32 seconds +[Apr 26, 2026 - 00:53:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 496 : data loading time 0.00 +[Apr 26, 2026 - 00:53:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 496 : avg forward pass time 13.63 +[Apr 26, 2026 - 00:53:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 496 : backward pass time 11.53 +[Apr 26, 2026 - 00:53:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 496 : optimizer step time 0.98 +[Apr 26, 2026 - 00:53:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 496-------------------- +[Apr 26, 2026 - 00:53:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0333 iter count: 1.0 +[Apr 26, 2026 - 00:53:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0333 iter count: 1.0 +[Apr 26, 2026 - 00:53:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:53:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:54:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319216 43.425114 43.211807 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.714355 70.777344 70.525391 +util 98.750000 100.000000 96.000000 +[Apr 26, 2026 - 00:54:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 497 : avg iteration time 46.95 seconds +[Apr 26, 2026 - 00:54:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 497 : data loading time 0.00 +[Apr 26, 2026 - 00:54:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 497 : avg forward pass time 11.43 +[Apr 26, 2026 - 00:54:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 497 : backward pass time 11.55 +[Apr 26, 2026 - 00:54:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 497 : optimizer step time 0.98 +[Apr 26, 2026 - 00:54:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 497-------------------- +[Apr 26, 2026 - 00:54:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0274 iter count: 1.0 +[Apr 26, 2026 - 00:54:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0274 iter count: 1.0 +[Apr 26, 2026 - 00:54:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:54:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:55:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319242 43.425121 43.211899 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.514160 70.542969 70.427734 +util 96.000000 100.000000 93.000000 +[Apr 26, 2026 - 00:55:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 498 : avg iteration time 49.90 seconds +[Apr 26, 2026 - 00:55:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 498 : data loading time 0.00 +[Apr 26, 2026 - 00:55:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 498 : avg forward pass time 12.92 +[Apr 26, 2026 - 00:55:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 498 : backward pass time 11.53 +[Apr 26, 2026 - 00:55:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 498 : optimizer step time 0.98 +[Apr 26, 2026 - 00:55:15 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 498-------------------- +[Apr 26, 2026 - 00:55:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0313 iter count: 1.0 +[Apr 26, 2026 - 00:55:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0313 iter count: 1.0 +[Apr 26, 2026 - 00:55:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:55:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:56:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319263 43.425121 43.212029 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.543457 70.544922 70.542969 +util 98.000000 99.000000 95.000000 +[Apr 26, 2026 - 00:56:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 499 : avg iteration time 49.97 seconds +[Apr 26, 2026 - 00:56:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 499 : data loading time 0.00 +[Apr 26, 2026 - 00:56:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 499 : avg forward pass time 12.95 +[Apr 26, 2026 - 00:56:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 499 : backward pass time 11.53 +[Apr 26, 2026 - 00:56:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 499 : optimizer step time 1.00 +[Apr 26, 2026 - 00:56:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 499-------------------- +[Apr 26, 2026 - 00:56:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0331 iter count: 1.0 +[Apr 26, 2026 - 00:56:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0331 iter count: 1.0 +[Apr 26, 2026 - 00:56:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:56:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.318942 43.423698 43.212009 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.488770 135.519531 135.478516 +util 68.000000 75.000000 64.000000 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 500 : avg iteration time 172.23 seconds +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 500 : data loading time 0.00 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 500 : avg forward pass time 76.84 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 500 : backward pass time 9.15 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 500 : optimizer step time 0.34 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 500-------------------- +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 2775632.2791 iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5467 iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 2775632.2791iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.2261iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.8236 iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 9.2814 iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 3657727.5000iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 5.1919 iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 115277288.0000iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5467iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.2742 iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 698.1788iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.5156 iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 456460.5938iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.4235 iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 127.3544iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.6685 iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1535.9941iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0316 iter count: 1.0 +[Apr 26, 2026 - 00:58:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 26, 2026 - 00:59:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 15.71 seconds +[Apr 26, 2026 - 00:59:13 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 500 +[Apr 26, 2026 - 00:59:13 | INFO | fastgen.utils.checkpointer:save:301 ] Saving FSDP model to prefix /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints/0000500 +[Apr 26, 2026 - 00:59:28 | SUCCESS | fastgen.utils.checkpointer:save:329 ] Model saved at iteration 500 +[Apr 26, 2026 - 00:59:29 | INFO | fastgen.trainer:validate:358 ] Validation iteration 500 +[Apr 26, 2026 - 00:59:29 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 26, 2026 - 00:59:31 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 2.3s +[Apr 26, 2026 - 00:59:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 26, 2026 - 00:59:37 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 26, 2026 - 00:59:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 26, 2026 - 00:59:42 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 26, 2026 - 00:59:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 26, 2026 - 00:59:47 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 26, 2026 - 00:59:51 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 26, 2026 - 00:59:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 26, 2026 - 00:59:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 26, 2026 - 00:59:58 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 26, 2026 - 01:00:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 26, 2026 - 01:00:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 26, 2026 - 01:00:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 26, 2026 - 01:00:08 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 26, 2026 - 01:00:12 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 26, 2026 - 01:00:14 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 26, 2026 - 01:00:17 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 26, 2026 - 01:00:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 26, 2026 - 01:00:22 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging val0 stats at iteration 500-------------------- +[Apr 26, 2026 - 01:00:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 26, 2026 - 01:00:39 | INFO | fastgen.callbacks.wandb:on_validation_end:683 ] Logged 10 val videos at iteration 500 +[Apr 26, 2026 - 01:01:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319961 43.427807 43.211727 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.875488 71.169922 70.777344 +util 98.750000 100.000000 97.000000 +[Apr 26, 2026 - 01:01:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 501 : avg iteration time 153.51 seconds +[Apr 26, 2026 - 01:01:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 501 : data loading time 0.00 +[Apr 26, 2026 - 01:01:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 501 : avg forward pass time 14.00 +[Apr 26, 2026 - 01:01:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 501 : backward pass time 11.56 +[Apr 26, 2026 - 01:01:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 501 : optimizer step time 0.98 +[Apr 26, 2026 - 01:01:31 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 501-------------------- +[Apr 26, 2026 - 01:01:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0312 iter count: 1.0 +[Apr 26, 2026 - 01:01:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0312 iter count: 1.0 +[Apr 26, 2026 - 01:01:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:01:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:02:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319960 43.427780 43.211727 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.841309 71.150391 70.738281 +util 97.750000 100.000000 93.000000 +[Apr 26, 2026 - 01:02:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 502 : avg iteration time 50.04 seconds +[Apr 26, 2026 - 01:02:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 502 : data loading time 0.00 +[Apr 26, 2026 - 01:02:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 502 : avg forward pass time 12.96 +[Apr 26, 2026 - 01:02:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 502 : backward pass time 11.57 +[Apr 26, 2026 - 01:02:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 502 : optimizer step time 0.98 +[Apr 26, 2026 - 01:02:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 502-------------------- +[Apr 26, 2026 - 01:02:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0286 iter count: 1.0 +[Apr 26, 2026 - 01:02:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0286 iter count: 1.0 +[Apr 26, 2026 - 01:02:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:02:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:03:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.320051 43.428066 43.211727 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.787598 70.994141 70.718750 +util 97.500000 99.000000 93.000000 +[Apr 26, 2026 - 01:03:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 503 : avg iteration time 50.00 seconds +[Apr 26, 2026 - 01:03:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 503 : data loading time 0.00 +[Apr 26, 2026 - 01:03:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 503 : avg forward pass time 12.95 +[Apr 26, 2026 - 01:03:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 503 : backward pass time 11.54 +[Apr 26, 2026 - 01:03:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 503 : optimizer step time 0.98 +[Apr 26, 2026 - 01:03:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 503-------------------- +[Apr 26, 2026 - 01:03:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0286 iter count: 1.0 +[Apr 26, 2026 - 01:03:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0286 iter count: 1.0 +[Apr 26, 2026 - 01:03:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:03:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:04:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.320045 43.428043 43.211727 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.895020 71.306641 70.757812 +util 98.750000 100.000000 96.000000 +[Apr 26, 2026 - 01:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 504 : avg iteration time 52.95 seconds +[Apr 26, 2026 - 01:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 504 : data loading time 0.00 +[Apr 26, 2026 - 01:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 504 : avg forward pass time 14.45 +[Apr 26, 2026 - 01:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 504 : backward pass time 11.53 +[Apr 26, 2026 - 01:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 504 : optimizer step time 0.98 +[Apr 26, 2026 - 01:04:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 504-------------------- +[Apr 26, 2026 - 01:04:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0294 iter count: 1.0 +[Apr 26, 2026 - 01:04:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0294 iter count: 1.0 +[Apr 26, 2026 - 01:04:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:04:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319397 43.424072 43.212200 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.453125 135.500000 135.437500 +util 68.250000 73.000000 63.000000 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 505 : avg iteration time 172.97 seconds +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 505 : data loading time 0.00 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 505 : avg forward pass time 77.20 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 505 : backward pass time 9.17 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 505 : optimizer step time 0.34 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 505-------------------- +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 45970.0015 iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5405 iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 45970.0015iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5239iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.0778 iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.7962 iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 80553.8125iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 63.7951iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 799971.8750iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5405iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.0634 iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 457.9756iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.1295 iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 522.6974iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.5446 iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 8859.6797iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.0778 iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 63.7951 iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0353 iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:06:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 26, 2026 - 01:07:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319430 43.424118 43.212273 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.802246 70.835938 70.701172 +util 96.500000 99.000000 94.000000 +[Apr 26, 2026 - 01:07:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 506 : avg iteration time 51.55 seconds +[Apr 26, 2026 - 01:07:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 506 : data loading time 0.00 +[Apr 26, 2026 - 01:07:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 506 : avg forward pass time 13.76 +[Apr 26, 2026 - 01:07:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 506 : backward pass time 11.53 +[Apr 26, 2026 - 01:07:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 506 : optimizer step time 0.98 +[Apr 26, 2026 - 01:07:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 506-------------------- +[Apr 26, 2026 - 01:07:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0309 iter count: 1.0 +[Apr 26, 2026 - 01:07:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0309 iter count: 1.0 +[Apr 26, 2026 - 01:07:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:07:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:08:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319855 43.425827 43.212250 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.846191 70.894531 70.701172 +util 96.750000 100.000000 92.000000 +[Apr 26, 2026 - 01:08:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 507 : avg iteration time 50.04 seconds +[Apr 26, 2026 - 01:08:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 507 : data loading time 0.00 +[Apr 26, 2026 - 01:08:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 507 : avg forward pass time 13.00 +[Apr 26, 2026 - 01:08:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 507 : backward pass time 11.52 +[Apr 26, 2026 - 01:08:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 507 : optimizer step time 1.00 +[Apr 26, 2026 - 01:08:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 507-------------------- +[Apr 26, 2026 - 01:08:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0309 iter count: 1.0 +[Apr 26, 2026 - 01:08:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0309 iter count: 1.0 +[Apr 26, 2026 - 01:08:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:08:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:09:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319918 43.426056 43.212250 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.753418 70.777344 70.681641 +util 96.500000 99.000000 92.000000 +[Apr 26, 2026 - 01:09:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 508 : avg iteration time 50.00 seconds +[Apr 26, 2026 - 01:09:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 508 : data loading time 0.00 +[Apr 26, 2026 - 01:09:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 508 : avg forward pass time 12.98 +[Apr 26, 2026 - 01:09:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 508 : backward pass time 11.52 +[Apr 26, 2026 - 01:09:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 508 : optimizer step time 0.98 +[Apr 26, 2026 - 01:09:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 508-------------------- +[Apr 26, 2026 - 01:09:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0266 iter count: 1.0 +[Apr 26, 2026 - 01:09:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0266 iter count: 1.0 +[Apr 26, 2026 - 01:09:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:09:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:10:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319931 43.426056 43.212250 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.851074 70.894531 70.720703 +util 98.250000 100.000000 95.000000 +[Apr 26, 2026 - 01:10:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 509 : avg iteration time 52.83 seconds +[Apr 26, 2026 - 01:10:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 509 : data loading time 0.00 +[Apr 26, 2026 - 01:10:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 509 : avg forward pass time 14.41 +[Apr 26, 2026 - 01:10:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 509 : backward pass time 11.50 +[Apr 26, 2026 - 01:10:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 509 : optimizer step time 0.99 +[Apr 26, 2026 - 01:10:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 509-------------------- +[Apr 26, 2026 - 01:10:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0322 iter count: 1.0 +[Apr 26, 2026 - 01:10:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0322 iter count: 1.0 +[Apr 26, 2026 - 01:10:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:10:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319601 43.424519 43.212467 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.526855 135.537109 135.500000 +util 70.250000 75.000000 63.000000 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 510 : avg iteration time 170.25 seconds +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 510 : data loading time 0.00 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 510 : avg forward pass time 75.85 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 510 : backward pass time 9.13 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 510 : optimizer step time 0.34 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 510-------------------- +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 168567.4590 iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5167 iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 168567.4590iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.6182iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.8894 iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.6227 iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 321590.9375iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 5.9230 iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 4178251.0000iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5167iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.2347 iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 4766.6929iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.3641 iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 45625.5781iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.4450 iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 53638.9023iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.5035 iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 8159.3198iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0323 iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:13:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 26, 2026 - 01:14:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319712 43.424873 43.212444 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.714355 70.738281 70.642578 +util 97.500000 99.000000 93.000000 +[Apr 26, 2026 - 01:14:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 511 : avg iteration time 54.41 seconds +[Apr 26, 2026 - 01:14:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 511 : data loading time 0.00 +[Apr 26, 2026 - 01:14:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 511 : avg forward pass time 15.18 +[Apr 26, 2026 - 01:14:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 511 : backward pass time 11.54 +[Apr 26, 2026 - 01:14:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 511 : optimizer step time 0.97 +[Apr 26, 2026 - 01:14:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 511-------------------- +[Apr 26, 2026 - 01:14:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0315 iter count: 1.0 +[Apr 26, 2026 - 01:14:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0315 iter count: 1.0 +[Apr 26, 2026 - 01:14:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:14:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:14:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.320077 43.426132 43.212543 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.724121 70.757812 70.623047 +util 97.500000 99.000000 93.000000 +[Apr 26, 2026 - 01:14:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 512 : avg iteration time 49.92 seconds +[Apr 26, 2026 - 01:14:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 512 : data loading time 0.00 +[Apr 26, 2026 - 01:14:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 512 : avg forward pass time 12.94 +[Apr 26, 2026 - 01:14:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 512 : backward pass time 11.51 +[Apr 26, 2026 - 01:14:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 512 : optimizer step time 0.99 +[Apr 26, 2026 - 01:14:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 512-------------------- +[Apr 26, 2026 - 01:14:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0324 iter count: 1.0 +[Apr 26, 2026 - 01:14:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0324 iter count: 1.0 +[Apr 26, 2026 - 01:14:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:14:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:15:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.320114 43.426132 43.212673 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.724121 70.757812 70.623047 +util 96.250000 99.000000 90.000000 +[Apr 26, 2026 - 01:15:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 513 : avg iteration time 47.04 seconds +[Apr 26, 2026 - 01:15:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 513 : data loading time 0.00 +[Apr 26, 2026 - 01:15:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 513 : avg forward pass time 11.49 +[Apr 26, 2026 - 01:15:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 513 : backward pass time 11.54 +[Apr 26, 2026 - 01:15:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 513 : optimizer step time 0.99 +[Apr 26, 2026 - 01:15:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 513-------------------- +[Apr 26, 2026 - 01:15:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0291 iter count: 1.0 +[Apr 26, 2026 - 01:15:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0291 iter count: 1.0 +[Apr 26, 2026 - 01:15:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:15:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:16:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319873 43.426132 43.212654 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.602051 70.662109 70.582031 +util 97.000000 99.000000 92.000000 +[Apr 26, 2026 - 01:16:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 514 : avg iteration time 47.01 seconds +[Apr 26, 2026 - 01:16:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 514 : data loading time 0.00 +[Apr 26, 2026 - 01:16:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 514 : avg forward pass time 11.49 +[Apr 26, 2026 - 01:16:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 514 : backward pass time 11.52 +[Apr 26, 2026 - 01:16:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 514 : optimizer step time 0.99 +[Apr 26, 2026 - 01:16:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 514-------------------- +[Apr 26, 2026 - 01:16:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0280 iter count: 1.0 +[Apr 26, 2026 - 01:16:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0280 iter count: 1.0 +[Apr 26, 2026 - 01:16:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:16:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319568 43.424931 43.212631 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.488770 135.498047 135.460938 +util 70.000000 75.000000 64.000000 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 515 : avg iteration time 172.97 seconds +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 515 : data loading time 0.00 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 515 : avg forward pass time 77.20 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 515 : backward pass time 9.14 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 515 : optimizer step time 0.35 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 515-------------------- +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 33111.9739 iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5060 iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 33111.9739iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.9307iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.1665 iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.5193 iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 49765.7344iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 10.3088iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 459859.9062iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5060iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.5193 iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 459859.9062iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.1865 iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 236311.4219iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.7324 iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1745.4396iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.7048 iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 12204.9795iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0314 iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:19:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 26, 2026 - 01:20:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319581 43.424938 43.212646 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.645996 70.679688 70.544922 +util 97.000000 100.000000 92.000000 +[Apr 26, 2026 - 01:20:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 516 : avg iteration time 54.36 seconds +[Apr 26, 2026 - 01:20:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 516 : data loading time 0.00 +[Apr 26, 2026 - 01:20:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 516 : avg forward pass time 15.14 +[Apr 26, 2026 - 01:20:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 516 : backward pass time 11.56 +[Apr 26, 2026 - 01:20:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 516 : optimizer step time 0.97 +[Apr 26, 2026 - 01:20:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 516-------------------- +[Apr 26, 2026 - 01:20:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0414 iter count: 1.0 +[Apr 26, 2026 - 01:20:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0414 iter count: 1.0 +[Apr 26, 2026 - 01:20:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:20:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:21:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319899 43.426178 43.212666 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.641113 70.660156 70.583984 +util 96.750000 99.000000 94.000000 +[Apr 26, 2026 - 01:21:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 517 : avg iteration time 47.16 seconds +[Apr 26, 2026 - 01:21:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 517 : data loading time 0.00 +[Apr 26, 2026 - 01:21:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 517 : avg forward pass time 11.53 +[Apr 26, 2026 - 01:21:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 517 : backward pass time 11.54 +[Apr 26, 2026 - 01:21:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 517 : optimizer step time 0.99 +[Apr 26, 2026 - 01:21:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 517-------------------- +[Apr 26, 2026 - 01:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0340 iter count: 1.0 +[Apr 26, 2026 - 01:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0340 iter count: 1.0 +[Apr 26, 2026 - 01:21:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:21:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:21:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319937 43.426182 43.212646 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.611816 70.621094 70.583984 +util 96.500000 99.000000 92.000000 +[Apr 26, 2026 - 01:21:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 518 : avg iteration time 47.04 seconds +[Apr 26, 2026 - 01:21:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 518 : data loading time 0.00 +[Apr 26, 2026 - 01:21:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 518 : avg forward pass time 11.46 +[Apr 26, 2026 - 01:21:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 518 : backward pass time 11.55 +[Apr 26, 2026 - 01:21:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 518 : optimizer step time 0.98 +[Apr 26, 2026 - 01:21:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 518-------------------- +[Apr 26, 2026 - 01:21:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0334 iter count: 1.0 +[Apr 26, 2026 - 01:21:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0334 iter count: 1.0 +[Apr 26, 2026 - 01:21:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:21:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:22:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319958 43.426258 43.212646 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.616699 70.621094 70.603516 +util 97.250000 99.000000 95.000000 +[Apr 26, 2026 - 01:22:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 519 : avg iteration time 49.98 seconds +[Apr 26, 2026 - 01:22:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 519 : data loading time 0.00 +[Apr 26, 2026 - 01:22:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 519 : avg forward pass time 12.96 +[Apr 26, 2026 - 01:22:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 519 : backward pass time 11.53 +[Apr 26, 2026 - 01:22:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 519 : optimizer step time 0.98 +[Apr 26, 2026 - 01:22:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 519-------------------- +[Apr 26, 2026 - 01:22:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0296 iter count: 1.0 +[Apr 26, 2026 - 01:22:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0296 iter count: 1.0 +[Apr 26, 2026 - 01:22:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:22:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319772 43.425365 43.212711 +peak_gpu_mem_gb 131.568343 131.569888 131.567828 +peak_gpu_mem_reserved_gb 135.502930 135.517578 135.498047 +util 68.000000 72.000000 63.000000 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 520 : avg iteration time 179.24 seconds +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 520 : data loading time 0.00 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 520 : avg forward pass time 80.30 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 520 : backward pass time 9.17 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 520 : optimizer step time 0.35 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 520-------------------- +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 2511255.4927 iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4611 iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 2511255.4927iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.6708iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5556 iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 9.0640 iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 3571392.2500iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.0380 iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 74630184.0000iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4611iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.2916 iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 97.8210 iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.2840 iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 711.9233iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.7004 iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 12098.5723iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.4976 iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 440331.5000iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0361 iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:25:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 26, 2026 - 01:26:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319793 43.425354 43.212818 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.645996 70.779297 70.601562 +util 97.750000 99.000000 94.000000 +[Apr 26, 2026 - 01:26:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 521 : avg iteration time 51.48 seconds +[Apr 26, 2026 - 01:26:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 521 : data loading time 0.00 +[Apr 26, 2026 - 01:26:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 521 : avg forward pass time 13.71 +[Apr 26, 2026 - 01:26:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 521 : backward pass time 11.54 +[Apr 26, 2026 - 01:26:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 521 : optimizer step time 0.98 +[Apr 26, 2026 - 01:26:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 521-------------------- +[Apr 26, 2026 - 01:26:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0334 iter count: 1.0 +[Apr 26, 2026 - 01:26:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0334 iter count: 1.0 +[Apr 26, 2026 - 01:26:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:26:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:27:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319921 43.425869 43.212818 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.626465 70.640625 70.583984 +util 95.750000 99.000000 91.000000 +[Apr 26, 2026 - 01:27:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 522 : avg iteration time 52.90 seconds +[Apr 26, 2026 - 01:27:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 522 : data loading time 0.00 +[Apr 26, 2026 - 01:27:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 522 : avg forward pass time 14.44 +[Apr 26, 2026 - 01:27:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 522 : backward pass time 11.51 +[Apr 26, 2026 - 01:27:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 522 : optimizer step time 0.99 +[Apr 26, 2026 - 01:27:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 522-------------------- +[Apr 26, 2026 - 01:27:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0349 iter count: 1.0 +[Apr 26, 2026 - 01:27:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0349 iter count: 1.0 +[Apr 26, 2026 - 01:27:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:27:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:28:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.319942 43.425926 43.212818 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.729004 70.777344 70.583984 +util 95.500000 100.000000 92.000000 +[Apr 26, 2026 - 01:28:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 523 : avg iteration time 46.95 seconds +[Apr 26, 2026 - 01:28:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 523 : data loading time 0.00 +[Apr 26, 2026 - 01:28:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 523 : avg forward pass time 11.46 +[Apr 26, 2026 - 01:28:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 523 : backward pass time 11.50 +[Apr 26, 2026 - 01:28:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 523 : optimizer step time 0.98 +[Apr 26, 2026 - 01:28:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 523-------------------- +[Apr 26, 2026 - 01:28:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0361 iter count: 1.0 +[Apr 26, 2026 - 01:28:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0361 iter count: 1.0 +[Apr 26, 2026 - 01:28:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:28:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/requirements.txt b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/wandb-metadata.json b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2f120b13893eaabffaf4e13063d5c8710065cc27 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/wandb-metadata.json @@ -0,0 +1,73 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-04-25T05:26:02.550147Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_sink1_window7_redmd_beta2_taew.py", + "-", + "trainer.resume=False", + "model.fake_score_optimizer.lr=3e-6", + "log_config.group=omniavatar_sf_audiofix", + "log_config.name=sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho", + "model.net.sink_size=3", + "model.net.local_attn_size=9" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "31752c6328ad798bdbb8f20a8372fca5fb35b990" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "1097760292864" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "d8umwtq7xb7xzsinoavf8csm2fyls44w" +} \ No newline at end of file diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/wandb-summary.json b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..494224df52e0bbc71c953630c6f8be78d41ce6b8 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/wandb-summary.json @@ -0,0 +1 @@ +{"_runtime":39754.161806512,"val0/reconstructed":{"videos":[{"sha256":"80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327","size":182028,"path":"media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4","_type":"video-file"},{"size":170922,"path":"media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4","_type":"video-file","sha256":"884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214"},{"size":173236,"path":"media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4","_type":"video-file","sha256":"c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8"},{"_type":"video-file","sha256":"294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c","size":249372,"path":"media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4"},{"size":190004,"path":"media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4","_type":"video-file","sha256":"15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117"},{"_type":"video-file","sha256":"f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395","size":164630,"path":"media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4"},{"sha256":"c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8","size":222986,"path":"media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4","_type":"video-file"},{"_type":"video-file","sha256":"a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb","size":165516,"path":"media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4"},{"size":214001,"path":"media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4","_type":"video-file","sha256":"6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac"},{"_type":"video-file","sha256":"f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273","size":227490,"path":"media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4"}],"captions":false,"_type":"videos","count":10},"train/gan_loss_gen":0,"net/local_total_params":[376675360,376398880,376398880,376398880],"fake_score/local_total_params":[399300904,399021352,399021352,399021352],"train/vsd_loss_weighted":2.5112554926757812e+06,"net/total_params":1421383840,"model/local_total_params":[4364057584,4362262512,4362262512,4362262512],"optimizer/non_finite_grads_count (model_key net)":0,"train/reward_sync_c_mean":4.670807361602783,"optimizer/lr_net":2e-06,"fake_score/trainable_params":174981120,"train/reward_weight_max":7.4630184e+07,"fake_score/local_trainable_params":[43745280,43745280,43745280,43745280],"model/local_trainable_params":[420420640,420144160,420144160,420144160],"train/reward_weight_r0":97.82101440429688,"val0/total_loss":0,"fake_score/total_params":1596364960,"train/reward_sync_c_r3":6.497641563415527,"train/reward_weight_r1":711.92333984375,"train_media/student/generation":{"size":346634,"path":"media/videos/train_media/student/generation_500_67ac3650ef016bc0d866.mp4","_type":"video-file","sha256":"67ac3650ef016bc0d866bc3059b250b55dd12e4f8f3a8467c7d2b0a02ca1a353"},"_step":523,"train/reward_sync_c_r0":2.291569709777832,"optimizer/lr_fake_score":3e-06,"optimizer/iteration":523,"train/vsd_loss_unweighted":0.4610595703125,"train/gan_loss_disc":0,"_timestamp":1.7771344923590834e+09,"train/reward_weight_min":3.0380027294158936,"profiler/data_loading_time":0.0009152963757514954,"train/total_loss":0.03607177734375,"profiler/avg_iteration_time":46.95137691497803,"optimizer/grad_norm (model_key net)":0,"model/total_params":17325462112,"train/reward_sync_c_max":9.064027786254883,"train_media/data/real":{"path":"media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4","_type":"video-file","sha256":"2594e32c120a620ede9d4613c87c8985d5ec03a9250077d94093cf8001922aae","size":190461},"model/trainable_params":1596364960,"net/local_trainable_params":[376675360,376398880,376398880,376398880],"train/reward_weight_r3":440331.5,"train/reward_sync_c_r1":3.283985137939453,"val0/generated":{"_type":"videos","count":10,"videos":[{"_type":"video-file","sha256":"b6e3fb89f026804ddf554a8a2ca65808354af72d08ae6779be0ed727eaf08d00","size":336122,"path":"media/videos/val0/generated_500_b6e3fb89f026804ddf55.mp4"},{"_type":"video-file","sha256":"364210d1d39487ed18dd123ed07f223616c35ace4fd3bdfab872a29b20c0aded","size":372058,"path":"media/videos/val0/generated_500_364210d1d39487ed18dd.mp4"},{"_type":"video-file","sha256":"382dcc9d28af35e517dd747d8e08ecf5123327ce5699fc118dbe1e53a4261191","size":320950,"path":"media/videos/val0/generated_500_382dcc9d28af35e517dd.mp4"},{"_type":"video-file","sha256":"b22bbc588eb2412872cb360c8340ac354f0063cbffc58c5b8fc61407a74b6277","size":442827,"path":"media/videos/val0/generated_500_b22bbc588eb2412872cb.mp4"},{"_type":"video-file","sha256":"415f82bb3b3e9cd393a095c95503284e21ed8de81292a042499f9cc5a716a9be","size":405114,"path":"media/videos/val0/generated_500_415f82bb3b3e9cd393a0.mp4"},{"path":"media/videos/val0/generated_500_e2e0efb6d3340c8a74b5.mp4","_type":"video-file","sha256":"e2e0efb6d3340c8a74b58b9d1b68b7a050660b6658f2f789686d51bc520a4f9c","size":290175},{"_type":"video-file","sha256":"90b0c03047027618dec650a0421971be47de5dbbfa44396f5e38eac4fdee9704","size":477909,"path":"media/videos/val0/generated_500_90b0c03047027618dec6.mp4"},{"path":"media/videos/val0/generated_500_52ceb51262f49782f4b9.mp4","_type":"video-file","sha256":"52ceb51262f49782f4b91466896ee5be6c5c1c6e2e83f5f16137e78ff1278f9a","size":332242},{"_type":"video-file","sha256":"08b64e302ec053e09f7ad36f87a71ccc71a553a8f2445caf8724efe85e14ffbe","size":358062,"path":"media/videos/val0/generated_500_08b64e302ec053e09f7a.mp4"},{"size":377778,"path":"media/videos/val0/generated_500_f565d15af15a1ba1d0af.mp4","_type":"video-file","sha256":"f565d15af15a1ba1d0af3054dfa7270010be5130908fc3e411426ef37525df7a"}],"captions":false},"profiler/backward_pass_time":11.502201883122325,"train/reward_weight_r2":12098.572265625,"train/fake_score_loss":0.03607177734375,"train/reward_weight_mean":3.57139225e+06,"train/reward_sync_c_min":0.5556001663208008,"profiler/avg_forward_pass_time":11.462399146519601,"train/vsd_loss":0.4610595703125,"profiler/optimizer_step_time":0.9770041685551405,"_wandb":{"runtime":39754},"train/reward_sync_c_r2":4.700421333312988,"net/trainable_params":1421383840} \ No newline at end of file diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/logs/debug-core.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..ea478dcdd37df9bb7895f0eb4c7925b45bcae459 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2026-04-25T14:26:02.602938304+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp816do_th/port-953349.txt","pid":953349,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-04-25T14:26:02.603474093+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":953349} +{"time":"2026-04-25T14:26:02.603442622+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-953349-960353-870093847/socket","Net":"unix"}} +{"time":"2026-04-25T14:26:02.785525751+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-04-25T14:26:02.799761138+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"3fi9xe8g","id":"1(@)"} +{"time":"2026-04-25T14:26:03.186572267+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"3fi9xe8g","id":"1(@)"} +{"time":"2026-04-25T14:26:09.228377999+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"xxsut9s41tf3"} +{"time":"2026-04-26T01:28:38.020787197+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-04-26T01:28:38.020972191+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-04-26T01:28:38.020963588+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-04-26T01:28:38.0210444+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-04-26T01:28:38.021042128+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-953349-960353-870093847/socket","Net":"unix"}} +{"time":"2026-04-26T01:28:38.30553962+09:00","level":"INFO","msg":"server: parent process exited, terminating service process"} diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/logs/debug-internal.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4e2bc57ab266dbaf76bdf43543f8df58b6d0fbd2 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/logs/debug-internal.log @@ -0,0 +1,7 @@ +{"time":"2026-04-25T14:26:02.799847511+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-04-25T14:26:03.186449198+09:00","level":"INFO","msg":"stream: created new stream","id":"3fi9xe8g"} +{"time":"2026-04-25T14:26:03.186497211+09:00","level":"INFO","msg":"handler: started","stream_id":"3fi9xe8g"} +{"time":"2026-04-25T14:26:03.186565827+09:00","level":"INFO","msg":"stream: started","id":"3fi9xe8g"} +{"time":"2026-04-25T14:26:03.186588125+09:00","level":"INFO","msg":"sender: started","stream_id":"3fi9xe8g"} +{"time":"2026-04-25T14:26:03.186587608+09:00","level":"INFO","msg":"writer: started","stream_id":"3fi9xe8g"} +{"time":"2026-04-26T01:28:38.020968115+09:00","level":"INFO","msg":"stream: closing","id":"3fi9xe8g"} diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/logs/debug.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..60ac9f2e459e930481062941e7a4246d4df9506a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/logs/debug.log @@ -0,0 +1,21 @@ +2026-04-25 14:26:02,551 INFO MainThread:953349 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-04-25 14:26:02,551 INFO MainThread:953349 [wandb_setup.py:_flush():81] Configure stats pid to 953349 +2026-04-25 14:26:02,551 INFO MainThread:953349 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-25 14:26:02,551 INFO MainThread:953349 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/logs/debug.log +2026-04-25 14:26:02,551 INFO MainThread:953349 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/logs/debug-internal.log +2026-04-25 14:26:02,551 INFO MainThread:953349 [wandb_init.py:init():844] calling init triggers +2026-04-25 14:26:02,551 INFO MainThread:953349 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_audiofix', 'name': 'sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_100iter/checkpoints/0000100.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '5000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': None, 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': None, 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '9', 'sink_size': '3', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'False', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.833', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '3e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'True', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '2', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-04-25 14:26:02,551 INFO MainThread:953349 [wandb_init.py:init():892] starting backend +2026-04-25 14:26:02,785 INFO MainThread:953349 [wandb_init.py:init():895] sending inform_init request +2026-04-25 14:26:02,797 INFO MainThread:953349 [wandb_init.py:init():903] backend started and connected +2026-04-25 14:26:02,801 INFO MainThread:953349 [wandb_init.py:init():973] updated telemetry +2026-04-25 14:26:02,825 INFO MainThread:953349 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-25 14:26:03,857 INFO MainThread:953349 [wandb_init.py:init():1042] starting run threads in backend +2026-04-25 14:26:04,224 INFO MainThread:953349 [wandb_run.py:_console_start():2524] atexit reg +2026-04-25 14:26:04,225 INFO MainThread:953349 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-04-25 14:26:04,225 INFO MainThread:953349 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-04-25 14:26:04,225 INFO MainThread:953349 [wandb_run.py:_redirect():2465] Redirects installed. +2026-04-25 14:26:04,232 INFO MainThread:953349 [wandb_init.py:init():1082] run started, returning control to user process +2026-04-26 01:28:38,020 INFO wandb-AsyncioManager-main:953349 [service_client.py:_forward_responses():134] Reached EOF. +2026-04-26 01:28:38,021 INFO wandb-AsyncioManager-main:953349 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/run-3fi9xe8g.wandb b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/run-3fi9xe8g.wandb new file mode 100644 index 0000000000000000000000000000000000000000..0ae7951fa05a30a1c968ed8d19e5d8ecafa65fc6 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/run-3fi9xe8g.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e992e28513d73a0bbd8fc08e3aa4ea8054b77c5342096f9722468065545dc520 +size 8978432 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb_id.txt b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb_id.txt new file mode 100644 index 0000000000000000000000000000000000000000..ae987f4caac41eaeb4b905e39352102b526135f7 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb_id.txt @@ -0,0 +1 @@ +3fi9xe8g