Add FASTGEN_SF_OUTPUT

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_5000iter_shift5/config.yaml

@@ -0,0 +1,259 @@
+dataloader_train:
+  _target_: <class 'fastgen.datasets.omniavatar_dataloader.OmniAvatarDataLoader'>
+  batch_size: '8'
+  data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt
+  latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png
+  neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt
+  num_workers: '2'
+  use_ref_sequence: 'True'
+dataloader_val:
+  _target_: <function create_omniavatar_dataloader at 0x7fa0fef8fb00>
+  batch_size: '1'
+  data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt
+  latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png
+  load_ode_path: 'False'
+  neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt
+  num_workers: '2'
+  use_ref_sequence: 'True'
+eval:
+  max_ckpt: '100000000'
+  min_ckpt: '0'
+  num_samples: '50000'
+  samples_dir: samples
+  save_images: 'False'
+log_config:
+  group: omniavatar_sf
+  name: sf_4gpu_bs8_5000iter_shift5
+  project: OmniAvatar-FastGen
+  wandb_credential: ./credentials/wandb_api.txt
+  wandb_entity: paulhcho
+  wandb_mode: online
+model:
+  add_teacher_to_fsdp_dict: 'True'
+  context_noise: '0.0'
+  ddp_find_unused_parameters: 'True'
+  device: cuda
+  discriminator:
+    _target_: <class 'fastgen.networks.discriminators.Discriminator_EDM'>
+    all_res:
+    - '32'
+    - '16'
+    - '8'
+    feature_indices: '{0, 1, 2}'
+    in_channels: '256'
+  discriminator_optimizer:
+    _target_: <function get_optimizer at 0x7fa0fefee660>
+    betas:
+    - '0.9'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: '0.0001'
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  discriminator_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  enable_gradient_in_rollout: 'True'
+  enable_preprocessors: 'True'
+  fake_score: null
+  fake_score_net:
+    _target_: <class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors
+    in_dim: '65'
+    merge_lora: 'False'
+    mode: v2v
+    model_size: 1.3B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt
+    schedule_type: rf
+    use_audio: 'True'
+  fake_score_optimizer:
+    _target_: <function get_optimizer at 0x7fa0fefee660>
+    betas:
+    - '0.0'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: 2e-06
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  fake_score_pred_type: x0
+  fake_score_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  fsdp_meta_init: 'False'
+  gan_loss_weight_gen: '0'
+  gan_r1_reg_alpha: '0.1'
+  gan_r1_reg_weight: '0.0'
+  gan_use_same_t_noise: 'False'
+  grad_scaler_enabled: 'False'
+  grad_scaler_growth_interval: '2000'
+  grad_scaler_init_scale: '65536.0'
+  guidance_scale: '4.5'
+  input_shape:
+  - '16'
+  - '21'
+  - '64'
+  - '64'
+  last_step_only: 'False'
+  load_student_weights: 'False'
+  net:
+    _target_: <class 'fastgen.networks.OmniAvatar.network_causal.CausalOmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors
+    chunk_size: '3'
+    in_dim: '65'
+    mode: v2v
+    model_size: 1.3B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt
+    schedule_type: rf
+    total_num_frames: '21'
+    use_audio: 'True'
+  net_optimizer:
+    _target_: <function get_optimizer at 0x7fa0fefee660>
+    betas:
+    - '0.0'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: 2e-06
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  net_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  precision: bfloat16
+  precision_amp: null
+  precision_amp_enc: null
+  precision_amp_infer: null
+  precision_fsdp: bfloat16
+  pretrained_model_path: ''
+  pretrained_student_net_path: ''
+  same_step_across_blocks: 'True'
+  sample_t_cfg:
+    log_t_df: '0.01'
+    max_t: '0.999'
+    min_t: '0.001'
+    shift: '5.0'
+    t_list:
+    - '0.999'
+    - '0.937'
+    - '0.833'
+    - '0.624'
+    - '0.0'
+    time_dist_type: shifted
+    train_p_mean: '-1.1'
+    train_p_std: '2.0'
+  skip_layers: null
+  start_gradient_frame: '0'
+  student_sample_steps: '4'
+  student_sample_type: sde
+  student_update_freq: '5'
+  teacher:
+    _target_: <class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors
+    in_dim: '65'
+    merge_lora: 'True'
+    mode: v2v
+    model_size: 14B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_phase2/step-10500.pt
+    schedule_type: rf
+    use_audio: 'True'
+  use_ema: 'False'
+model_class:
+  _target_: <class 'fastgen.methods.omniavatar_self_forcing.OmniAvatarSelfForcingModel'>
+  config: null
+trainer:
+  augment_pipe: null
+  batch_size_global: null
+  callbacks:
+    ema:
+      _target_: <class 'fastgen.callbacks.ema.EMACallback'>
+      beta: '0.9999'
+      ema_halflife_kimg: '500'
+      ema_rampup_ratio: '0.05'
+      gamma: '16.97'
+      start_iter: '0'
+      type: constant
+    gpu_stats:
+      _target_: <class 'fastgen.callbacks.gpu_stats.GPUStatsCallback'>
+      every_n: '100'
+    grad_clip:
+      _target_: <class 'fastgen.callbacks.grad_clip.GradClipCallback'>
+      grad_norm: '10.0'
+      model_key: net
+    param_count:
+      _target_: <class 'fastgen.callbacks.param_count.ParamCountCallback'>
+    train_profiler:
+      _target_: <class 'fastgen.callbacks.train_profiler.TrainProfilerCallback'>
+      every_n: '100'
+    wandb:
+      _target_: <class 'fastgen.callbacks.wandb.WandbCallback'>
+      fps: '25'
+      sample_logging_iter: '100'
+  checkpointer:
+    pretrained_ckpt_key_map:
+      net: net
+    pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_lr1e5_10000iter_shift_5/checkpoints/0005000.pth
+    s3_container: s3://checkpoints/fastgen
+    s3_credential: ./credentials/s3.json
+    save_dir: /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_5000iter_shift5/checkpoints
+    use_s3: 'False'
+  cudnn:
+    benchmark: 'True'
+    deterministic: 'False'
+  ddp: 'False'
+  fsdp: 'True'
+  fsdp_cpu_offload: 'False'
+  fsdp_min_num_params: '10000000'
+  fsdp_sharding_group_size: null
+  global_vars: null
+  global_vars_val:
+  - null
+  grad_accum_rounds: '2'
+  logging_iter: '1'
+  max_iter: '5000'
+  offload_module_in_decoding: 'False'
+  resume: 'False'
+  save_ckpt_iter: '100'
+  seed: '0'
+  skip_initial_validation: 'True'
+  tf32_enabled: 'True'
+  val_seed: null
+  validation_iter: '100'
+  visualize_teacher: 'False'

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_5000iter_shift5/wandb/debug-internal.log

@@ -0,0 +1,6 @@
+{"time":"2026-04-02T20:58:42.181919016+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
+{"time":"2026-04-02T20:58:42.574143607+09:00","level":"INFO","msg":"stream: created new stream","id":"zmgbhqqw"}
+{"time":"2026-04-02T20:58:42.574192032+09:00","level":"INFO","msg":"handler: started","stream_id":"zmgbhqqw"}
+{"time":"2026-04-02T20:58:42.574261225+09:00","level":"INFO","msg":"stream: started","id":"zmgbhqqw"}
+{"time":"2026-04-02T20:58:42.574281395+09:00","level":"INFO","msg":"writer: started","stream_id":"zmgbhqqw"}
+{"time":"2026-04-02T20:58:42.57428521+09:00","level":"INFO","msg":"sender: started","stream_id":"zmgbhqqw"}

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_5000iter_shift5/wandb/debug.log

@@ -0,0 +1,19 @@
+2026-04-02 20:58:41,930 INFO    MainThread:549927 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
+2026-04-02 20:58:41,930 INFO    MainThread:549927 [wandb_setup.py:_flush():81] Configure stats pid to 549927
+2026-04-02 20:58:41,930 INFO    MainThread:549927 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-02 20:58:41,931 INFO    MainThread:549927 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_5000iter_shift5/wandb/run-20260402_205841-zmgbhqqw/logs/debug.log
+2026-04-02 20:58:41,931 INFO    MainThread:549927 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_5000iter_shift5/wandb/run-20260402_205841-zmgbhqqw/logs/debug-internal.log
+2026-04-02 20:58:41,931 INFO    MainThread:549927 [wandb_init.py:init():844] calling init triggers
+2026-04-02 20:58:41,931 INFO    MainThread:549927 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf', 'name': 'sf_4gpu_bs8_5000iter_shift5', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_5000iter_shift5/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_lr1e5_10000iter_shift_5/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': "<class 'fastgen.callbacks.grad_clip.GradClipCallback'>"}, 'gpu_stats': {'every_n': '100', '_target_': "<class 'fastgen.callbacks.gpu_stats.GPUStatsCallback'>"}, 'train_profiler': {'every_n': '100', '_target_': "<class 'fastgen.callbacks.train_profiler.TrainProfilerCallback'>"}, 'param_count': {'_target_': "<class 'fastgen.callbacks.param_count.ParamCountCallback'>"}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': "<class 'fastgen.callbacks.ema.EMACallback'>"}, 'wandb': {'sample_logging_iter': '100', '_target_': "<class 'fastgen.callbacks.wandb.WandbCallback'>", 'fps': '25'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '5000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "<class 'fastgen.datasets.omniavatar_dataloader.OmniAvatarDataLoader'>"}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': '<function create_omniavatar_dataloader at 0x7fbeacc8fb00>'}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "<class 'fastgen.networks.OmniAvatar.network_causal.CausalOmniAvatarWan'>"}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_phase2/step-10500.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "<class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>"}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt', 'merge_lora': 'False', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "<class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>"}, 'guidance_scale': '4.5', 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7fbeacee6660>'}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.937', '0.833', '0.624', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '4', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'bfloat16', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7fbeacee6660>'}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': "<class 'fastgen.networks.discriminators.Discriminator_EDM'>"}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7fbeacee6660>'}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None}, 'model_class': {'config': None, '_target_': "<class 'fastgen.methods.omniavatar_self_forcing.OmniAvatarSelfForcingModel'>"}, '_wandb': {}}
+2026-04-02 20:58:41,931 INFO    MainThread:549927 [wandb_init.py:init():892] starting backend
+2026-04-02 20:58:42,167 INFO    MainThread:549927 [wandb_init.py:init():895] sending inform_init request
+2026-04-02 20:58:42,179 INFO    MainThread:549927 [wandb_init.py:init():903] backend started and connected
+2026-04-02 20:58:42,183 INFO    MainThread:549927 [wandb_init.py:init():973] updated telemetry
+2026-04-02 20:58:42,199 INFO    MainThread:549927 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-04-02 20:58:43,281 INFO    MainThread:549927 [wandb_init.py:init():1042] starting run threads in backend
+2026-04-02 20:58:43,509 INFO    MainThread:549927 [wandb_run.py:_console_start():2524] atexit reg
+2026-04-02 20:58:43,509 INFO    MainThread:549927 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-04-02 20:58:43,509 INFO    MainThread:549927 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-04-02 20:58:43,509 INFO    MainThread:549927 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-04-02 20:58:43,513 INFO    MainThread:549927 [wandb_init.py:init():1082] run started, returning control to user process

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_5000iter_shift5/wandb/run-20260402_205841-zmgbhqqw/files/requirements.txt

@@ -0,0 +1,222 @@
+fastgen==0.1.0
+nvitop==1.6.1
+ftfy==6.3.1
+braceexpand==0.1.7
+antlr4-python3-runtime==4.9.3
+webdataset==1.0.2
+sentry-sdk==2.53.0
+rdkit==2025.9.5
+python-dotenv==1.2.1
+proglog==0.1.12
+omegaconf==2.3.0
+narwhals==2.17.0
+loguru==0.7.3
+imageio-ffmpeg==0.6.0
+plotly==6.5.2
+moviepy==2.2.1
+hydra-core==1.3.2
+wandb==0.25.0
+fastgen==0.1.0
+packaging==25.0
+setuptools==80.10.2
+wheel==0.46.3
+pip==26.0.1
+webencodings==0.5.1
+pure_eval==0.2.3
+ptyprocess==0.7.0
+nvidia-ml-py==13.590.48
+nvidia-cusparselt-cu12==0.7.1
+mpmath==1.3.0
+fastjsonschema==2.21.2
+zipp==3.23.0
+xyzservices==2025.11.0
+widgetsnbextension==4.0.15
+websocket-client==1.9.0
+webcolors==25.10.0
+wcwidth==0.6.0
+urllib3==2.6.3
+uri-template==1.3.0
+tzdata==2025.3
+typing_extensions==4.15.0
+triton==3.6.0
+traitlets==5.14.3
+tqdm==4.67.3
+tornado==6.5.5
+tinycss2==1.4.0
+sympy==1.14.0
+soupsieve==2.8.3
+smmap==5.0.3
+six==1.16.0
+sentencepiece==0.2.1
+Send2Trash==2.1.0
+safetensors==0.7.0
+rpds-py==0.30.0
+rfc3986-validator==0.1.1
+regex==2026.2.28
+pyzmq==27.1.0
+PyYAML==6.0.3
+python-json-logger==4.0.0
+Pygments==2.19.2
+pycparser==3.0
+psutil==7.2.2
+protobuf==4.24.4
+prometheus_client==0.24.1
+platformdirs==4.9.4
+pillow==11.3.0
+pexpect==4.9.0
+parso==0.8.6
+pandocfilters==1.5.1
+nvidia-nvtx-cu12==12.8.90
+nvidia-nvshmem-cu12==3.4.5
+nvidia-nvjitlink-cu12==12.8.93
+nvidia-nccl-cu12==2.27.5
+nvidia-curand-cu12==10.3.9.90
+nvidia-cufile-cu12==1.13.1.3
+nvidia-cuda-runtime-cu12==12.8.90
+nvidia-cuda-nvrtc-cu12==12.8.93
+nvidia-cuda-cupti-cu12==12.8.90
+nvidia-cublas-cu12==12.8.4.1
+numpy==1.26.4
+networkx==3.6.1
+nest-asyncio==1.6.0
+mistune==3.2.0
+MarkupSafe==3.0.3
+lark==1.3.1
+jupyterlab_widgets==3.0.16
+jupyterlab_pygments==0.3.0
+jsonpointer==3.0.0
+json5==0.13.0
+jmespath==1.1.0
+idna==3.11
+hf-xet==1.4.2
+h11==0.16.0
+fsspec==2026.2.0
+fqdn==1.5.1
+filelock==3.25.2
+executing==2.2.1
+einops==0.8.2
+defusedxml==0.7.1
+decorator==5.2.1
+debugpy==1.8.20
+cuda-pathfinder==1.4.2
+comm==0.2.3
+click==8.3.1
+charset-normalizer==3.4.5
+certifi==2026.2.25
+bleach==6.3.0
+babel==2.18.0
+av==17.0.0
+attrs==25.4.0
+async-lru==2.2.0
+asttokens==3.0.1
+annotated-types==0.7.0
+typing-inspection==0.4.2
+terminado==0.18.1
+stack-data==0.6.3
+scipy==1.17.1
+rfc3987-syntax==1.1.0
+rfc3339-validator==0.1.4
+requests==2.32.5
+referencing==0.37.0
+python-dateutil==2.9.0.post0
+pydantic_core==2.41.5
+prompt_toolkit==3.0.52
+opencv-python-headless==4.11.0.86
+nvidia-cusparse-cu12==12.5.8.93
+nvidia-cufft-cu12==11.3.3.83
+nvidia-cudnn-cu12==9.10.2.21
+matplotlib-inline==0.2.1
+jupyter_core==5.9.1
+Jinja2==3.1.6
+jedi==0.19.2
+ipython_pygments_lexers==1.1.1
+importlib_metadata==8.7.1
+ImageIO==2.37.3
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==12.9.4
+contourpy==1.3.3
+cffi==2.0.0
+beautifulsoup4==4.14.3
+anyio==4.12.1
+soundfile==0.13.1
+pydantic==2.12.5
+nvidia-cusolver-cu12==11.7.3.90
+jupyter_server_terminals==0.5.4
+jupyter_client==8.8.0
+jsonschema-specifications==2025.9.1
+ipython==9.11.0
+httpx==0.28.1
+GitPython==3.1.46
+botocore==1.42.68
+bokeh==3.9.0
+arrow==1.4.0
+argon2-cffi-bindings==25.1.0
+torch==2.10.0
+s3transfer==0.16.0
+jsonschema==4.26.0
+isoduration==20.11.0
+ipywidgets==8.1.8
+ipykernel==7.2.0
+argon2-cffi==25.1.0
+torchvision==0.25.0
+nbformat==5.10.4
+jupyter-console==6.6.3
+boto3==1.42.68
+accelerate==1.13.0
+nbclient==0.10.4
+jupyter-events==0.12.0
+nbconvert==7.17.0
+jupyter_server==2.17.0
+notebook_shim==0.2.4
+jupyterlab_server==2.28.0
+jupyter-lsp==2.3.0
+jupyterlab==4.5.6
+notebook==7.5.5
+jupyter==1.1.1
+fastgen==0.1.0
+pandas==3.0.1
+shellingham==1.5.4
+mdurl==0.1.2
+annotated-doc==0.0.4
+markdown-it-py==4.0.0
+rich==14.3.3
+typer==0.24.1
+huggingface_hub==1.7.1
+timm==1.0.25
+tokenizers==0.22.2
+diffusers==0.37.0
+transformers==5.3.0
+peft==0.18.1
+easydict==1.13
+lmdb==2.2.0
+threadpoolctl==3.6.0
+soxr==1.0.0
+msgpack==1.1.2
+llvmlite==0.47.0
+lazy-loader==0.5
+joblib==1.5.3
+audioread==3.1.0
+scikit-learn==1.8.0
+pooch==1.9.0
+numba==0.65.0
+librosa==0.11.0
+simsimd==6.5.16
+flatbuffers==25.12.19
+tifffile==2026.3.3
+stringzilla==4.6.0
+pyparsing==3.3.2
+prettytable==3.17.0
+onnx==1.17.0
+kiwisolver==1.5.0
+fonttools==4.62.1
+Cython==3.2.4
+cycler==0.12.1
+scikit-image==0.26.0
+onnxruntime==1.24.4
+matplotlib==3.10.8
+albucore==0.0.24
+albumentations==2.0.8
+insightface==0.7.3
+kornia_rs==0.1.10
+kornia==0.8.2

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_5000iter_shift5/wandb/run-20260402_205841-zmgbhqqw/files/wandb-metadata.json

@@ -0,0 +1,68 @@
+{
+  "os":  "Linux-5.15.0-151-generic-x86_64-with-glibc2.39",
+  "python":  "CPython 3.12.12",
+  "startedAt":  "2026-04-02T11:58:41.929718Z",
+  "args":  [
+    "--config=fastgen/configs/experiments/OmniAvatar/config_sf.py",
+    "-",
+    "trainer.resume=False",
+    "log_config.name=sf_4gpu_bs8_5000iter_shift5",
+    "log_config.project=OmniAvatar-FastGen"
+  ],
+  "program":  "/home/work/.local/hyunbin/FastGen/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://paulcho98:@github.com/paulcho98/FastGen.git",
+    "commit":  "dead092792003faa07babff77ccd223af4ad9b11"
+  },
+  "email":  "paul.hyunbin@gmail.com",
+  "root":  "/tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_5000iter_shift5",
+  "host":  "main1",
+  "executable":  "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12",
+  "cpu_count":  112,
+  "cpu_count_logical":  224,
+  "gpu":  "NVIDIA H200",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "1356758433792",
+      "used":  "257961558016"
+    }
+  },
+  "memory":  {
+    "total":  "2163961778176"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c"
+    },
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262"
+    },
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790"
+    },
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0"
+    }
+  ],
+  "cudaVersion":  "12.9",
+  "writerId":  "n2ybi81tgd0arslahhy2n7g532wc0pja"
+}

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_5000iter_shift5/wandb/run-20260402_205841-zmgbhqqw/logs/debug-core.log

@@ -0,0 +1,8 @@
+{"time":"2026-04-02T20:58:41.981092557+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpwwrvdk3u/port-549927.txt","pid":549927,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-02T20:58:41.98157225+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":549927}
+{"time":"2026-04-02T20:58:41.981562902+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-549927-558790-3160372452/socket","Net":"unix"}}
+{"time":"2026-04-02T20:58:42.16745332+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-02T20:58:42.181823375+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"zmgbhqqw","id":"1(@)"}
+{"time":"2026-04-02T20:58:42.574268009+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"zmgbhqqw","id":"1(@)"}
+{"time":"2026-04-02T20:58:48.512451301+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"6lpqhxkfa0yx"}
+{"time":"2026-04-02T21:41:09.853455246+09:00","level":"INFO","msg":"server: parent process exited, terminating service process"}

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_5000iter_shift5/wandb/run-20260402_205841-zmgbhqqw/logs/debug-internal.log

@@ -0,0 +1,6 @@
+{"time":"2026-04-02T20:58:42.181919016+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
+{"time":"2026-04-02T20:58:42.574143607+09:00","level":"INFO","msg":"stream: created new stream","id":"zmgbhqqw"}
+{"time":"2026-04-02T20:58:42.574192032+09:00","level":"INFO","msg":"handler: started","stream_id":"zmgbhqqw"}
+{"time":"2026-04-02T20:58:42.574261225+09:00","level":"INFO","msg":"stream: started","id":"zmgbhqqw"}
+{"time":"2026-04-02T20:58:42.574281395+09:00","level":"INFO","msg":"writer: started","stream_id":"zmgbhqqw"}
+{"time":"2026-04-02T20:58:42.57428521+09:00","level":"INFO","msg":"sender: started","stream_id":"zmgbhqqw"}

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_5000iter_shift5/wandb/run-20260402_205841-zmgbhqqw/logs/debug.log

@@ -0,0 +1,19 @@
+2026-04-02 20:58:41,930 INFO    MainThread:549927 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
+2026-04-02 20:58:41,930 INFO    MainThread:549927 [wandb_setup.py:_flush():81] Configure stats pid to 549927
+2026-04-02 20:58:41,930 INFO    MainThread:549927 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-02 20:58:41,931 INFO    MainThread:549927 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_5000iter_shift5/wandb/run-20260402_205841-zmgbhqqw/logs/debug.log
+2026-04-02 20:58:41,931 INFO    MainThread:549927 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_5000iter_shift5/wandb/run-20260402_205841-zmgbhqqw/logs/debug-internal.log
+2026-04-02 20:58:41,931 INFO    MainThread:549927 [wandb_init.py:init():844] calling init triggers
+2026-04-02 20:58:41,931 INFO    MainThread:549927 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf', 'name': 'sf_4gpu_bs8_5000iter_shift5', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_5000iter_shift5/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_lr1e5_10000iter_shift_5/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': "<class 'fastgen.callbacks.grad_clip.GradClipCallback'>"}, 'gpu_stats': {'every_n': '100', '_target_': "<class 'fastgen.callbacks.gpu_stats.GPUStatsCallback'>"}, 'train_profiler': {'every_n': '100', '_target_': "<class 'fastgen.callbacks.train_profiler.TrainProfilerCallback'>"}, 'param_count': {'_target_': "<class 'fastgen.callbacks.param_count.ParamCountCallback'>"}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': "<class 'fastgen.callbacks.ema.EMACallback'>"}, 'wandb': {'sample_logging_iter': '100', '_target_': "<class 'fastgen.callbacks.wandb.WandbCallback'>", 'fps': '25'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '5000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "<class 'fastgen.datasets.omniavatar_dataloader.OmniAvatarDataLoader'>"}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': '<function create_omniavatar_dataloader at 0x7fbeacc8fb00>'}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "<class 'fastgen.networks.OmniAvatar.network_causal.CausalOmniAvatarWan'>"}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_phase2/step-10500.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "<class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>"}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt', 'merge_lora': 'False', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "<class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>"}, 'guidance_scale': '4.5', 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7fbeacee6660>'}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.937', '0.833', '0.624', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '4', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'bfloat16', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7fbeacee6660>'}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': "<class 'fastgen.networks.discriminators.Discriminator_EDM'>"}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7fbeacee6660>'}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None}, 'model_class': {'config': None, '_target_': "<class 'fastgen.methods.omniavatar_self_forcing.OmniAvatarSelfForcingModel'>"}, '_wandb': {}}
+2026-04-02 20:58:41,931 INFO    MainThread:549927 [wandb_init.py:init():892] starting backend
+2026-04-02 20:58:42,167 INFO    MainThread:549927 [wandb_init.py:init():895] sending inform_init request
+2026-04-02 20:58:42,179 INFO    MainThread:549927 [wandb_init.py:init():903] backend started and connected
+2026-04-02 20:58:42,183 INFO    MainThread:549927 [wandb_init.py:init():973] updated telemetry
+2026-04-02 20:58:42,199 INFO    MainThread:549927 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-04-02 20:58:43,281 INFO    MainThread:549927 [wandb_init.py:init():1042] starting run threads in backend
+2026-04-02 20:58:43,509 INFO    MainThread:549927 [wandb_run.py:_console_start():2524] atexit reg
+2026-04-02 20:58:43,509 INFO    MainThread:549927 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-04-02 20:58:43,509 INFO    MainThread:549927 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-04-02 20:58:43,509 INFO    MainThread:549927 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-04-02 20:58:43,513 INFO    MainThread:549927 [wandb_init.py:init():1082] run started, returning control to user process

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_5000iter_shift5/wandb/run-20260402_205841-zmgbhqqw/run-zmgbhqqw.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7d6105253ff0592490ce0e6460f37480df990ab3d55586489772eaeeb75f982
+size 688128

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_5000iter_shift5/wandb_id.txt

@@ -0,0 +1 @@
+zmgbhqqw

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined/config.yaml

@@ -0,0 +1,259 @@
+dataloader_train:
+  _target_: <class 'fastgen.datasets.omniavatar_dataloader.OmniAvatarDataLoader'>
+  batch_size: '8'
+  data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt
+  latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png
+  neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt
+  num_workers: '2'
+  use_ref_sequence: 'True'
+dataloader_val:
+  _target_: <function create_omniavatar_dataloader at 0x7fc7a89d7ce0>
+  batch_size: '1'
+  data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt
+  latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png
+  load_ode_path: 'False'
+  neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt
+  num_workers: '2'
+  use_ref_sequence: 'True'
+eval:
+  max_ckpt: '100000000'
+  min_ckpt: '0'
+  num_samples: '50000'
+  samples_dir: samples
+  save_images: 'False'
+log_config:
+  group: omniavatar_sf
+  name: sf_4gpu_bs8_lr2e6_5000iter_shift5_combined
+  project: OmniAvatar-FastGen
+  wandb_credential: ./credentials/wandb_api.txt
+  wandb_entity: paulhcho
+  wandb_mode: online
+model:
+  add_teacher_to_fsdp_dict: 'True'
+  context_noise: '0.0'
+  ddp_find_unused_parameters: 'True'
+  device: cuda
+  discriminator:
+    _target_: <class 'fastgen.networks.discriminators.Discriminator_EDM'>
+    all_res:
+    - '32'
+    - '16'
+    - '8'
+    feature_indices: '{0, 1, 2}'
+    in_channels: '256'
+  discriminator_optimizer:
+    _target_: <function get_optimizer at 0x7fc7a8c2e660>
+    betas:
+    - '0.9'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: '0.0001'
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  discriminator_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  enable_gradient_in_rollout: 'True'
+  enable_preprocessors: 'True'
+  fake_score: null
+  fake_score_net:
+    _target_: <class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors
+    in_dim: '65'
+    merge_lora: 'False'
+    mode: v2v
+    model_size: 1.3B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt
+    schedule_type: rf
+    use_audio: 'True'
+  fake_score_optimizer:
+    _target_: <function get_optimizer at 0x7fc7a8c2e660>
+    betas:
+    - '0.0'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: 2e-06
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  fake_score_pred_type: x0
+  fake_score_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  fsdp_meta_init: 'False'
+  gan_loss_weight_gen: '0'
+  gan_r1_reg_alpha: '0.1'
+  gan_r1_reg_weight: '0.0'
+  gan_use_same_t_noise: 'False'
+  grad_scaler_enabled: 'False'
+  grad_scaler_growth_interval: '2000'
+  grad_scaler_init_scale: '65536.0'
+  guidance_scale: '4.5'
+  input_shape:
+  - '16'
+  - '21'
+  - '64'
+  - '64'
+  last_step_only: 'False'
+  load_student_weights: 'False'
+  net:
+    _target_: <class 'fastgen.networks.OmniAvatar.network_causal.CausalOmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors
+    chunk_size: '3'
+    in_dim: '65'
+    mode: v2v
+    model_size: 1.3B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt
+    schedule_type: rf
+    total_num_frames: '21'
+    use_audio: 'True'
+  net_optimizer:
+    _target_: <function get_optimizer at 0x7fc7a8c2e660>
+    betas:
+    - '0.0'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: 2e-06
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  net_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  precision: bfloat16
+  precision_amp: null
+  precision_amp_enc: null
+  precision_amp_infer: null
+  precision_fsdp: bfloat16
+  pretrained_model_path: ''
+  pretrained_student_net_path: ''
+  same_step_across_blocks: 'True'
+  sample_t_cfg:
+    log_t_df: '0.01'
+    max_t: '0.999'
+    min_t: '0.001'
+    shift: '5.0'
+    t_list:
+    - '0.999'
+    - '0.937'
+    - '0.833'
+    - '0.624'
+    - '0.0'
+    time_dist_type: shifted
+    train_p_mean: '-1.1'
+    train_p_std: '2.0'
+  skip_layers: null
+  start_gradient_frame: '0'
+  student_sample_steps: '4'
+  student_sample_type: sde
+  student_update_freq: '5'
+  teacher:
+    _target_: <class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors
+    in_dim: '65'
+    merge_lora: 'True'
+    mode: v2v
+    model_size: 14B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_phase2/step-10500.pt
+    schedule_type: rf
+    use_audio: 'True'
+  use_ema: 'False'
+model_class:
+  _target_: <class 'fastgen.methods.omniavatar_self_forcing.OmniAvatarSelfForcingModel'>
+  config: null
+trainer:
+  augment_pipe: null
+  batch_size_global: null
+  callbacks:
+    ema:
+      _target_: <class 'fastgen.callbacks.ema.EMACallback'>
+      beta: '0.9999'
+      ema_halflife_kimg: '500'
+      ema_rampup_ratio: '0.05'
+      gamma: '16.97'
+      start_iter: '0'
+      type: constant
+    gpu_stats:
+      _target_: <class 'fastgen.callbacks.gpu_stats.GPUStatsCallback'>
+      every_n: '100'
+    grad_clip:
+      _target_: <class 'fastgen.callbacks.grad_clip.GradClipCallback'>
+      grad_norm: '10.0'
+      model_key: net
+    param_count:
+      _target_: <class 'fastgen.callbacks.param_count.ParamCountCallback'>
+    train_profiler:
+      _target_: <class 'fastgen.callbacks.train_profiler.TrainProfilerCallback'>
+      every_n: '100'
+    wandb:
+      _target_: <class 'fastgen.callbacks.wandb.WandbCallback'>
+      fps: '25'
+      sample_logging_iter: '100'
+  checkpointer:
+    pretrained_ckpt_key_map:
+      net: net
+    pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_lr1e5_10000iter_shift_5/checkpoints/0005000.pth
+    s3_container: s3://checkpoints/fastgen
+    s3_credential: ./credentials/s3.json
+    save_dir: /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined/checkpoints
+    use_s3: 'False'
+  cudnn:
+    benchmark: 'True'
+    deterministic: 'False'
+  ddp: 'False'
+  fsdp: 'True'
+  fsdp_cpu_offload: 'False'
+  fsdp_min_num_params: '10000000'
+  fsdp_sharding_group_size: null
+  global_vars: null
+  global_vars_val:
+  - null
+  grad_accum_rounds: '2'
+  logging_iter: '1'
+  max_iter: '5000'
+  offload_module_in_decoding: 'False'
+  resume: 'False'
+  save_ckpt_iter: '100'
+  seed: '0'
+  skip_initial_validation: 'True'
+  tf32_enabled: 'True'
+  val_seed: null
+  validation_iter: '100'
+  visualize_teacher: 'False'

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined/wandb/debug-internal.log

@@ -0,0 +1,6 @@
+{"time":"2026-04-03T00:01:32.430639522+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
+{"time":"2026-04-03T00:01:32.800089951+09:00","level":"INFO","msg":"stream: created new stream","id":"nkf4iovm"}
+{"time":"2026-04-03T00:01:32.800139938+09:00","level":"INFO","msg":"handler: started","stream_id":"nkf4iovm"}
+{"time":"2026-04-03T00:01:32.800233729+09:00","level":"INFO","msg":"stream: started","id":"nkf4iovm"}
+{"time":"2026-04-03T00:01:32.80025365+09:00","level":"INFO","msg":"sender: started","stream_id":"nkf4iovm"}
+{"time":"2026-04-03T00:01:32.800252986+09:00","level":"INFO","msg":"writer: started","stream_id":"nkf4iovm"}

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined/wandb/debug.log

@@ -0,0 +1,19 @@
+2026-04-03 00:01:32,169 INFO    MainThread:1047116 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
+2026-04-03 00:01:32,169 INFO    MainThread:1047116 [wandb_setup.py:_flush():81] Configure stats pid to 1047116
+2026-04-03 00:01:32,169 INFO    MainThread:1047116 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-03 00:01:32,169 INFO    MainThread:1047116 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined/wandb/run-20260403_000132-nkf4iovm/logs/debug.log
+2026-04-03 00:01:32,170 INFO    MainThread:1047116 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined/wandb/run-20260403_000132-nkf4iovm/logs/debug-internal.log
+2026-04-03 00:01:32,170 INFO    MainThread:1047116 [wandb_init.py:init():844] calling init triggers
+2026-04-03 00:01:32,170 INFO    MainThread:1047116 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf', 'name': 'sf_4gpu_bs8_lr2e6_5000iter_shift5_combined', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_lr1e5_10000iter_shift_5/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': "<class 'fastgen.callbacks.grad_clip.GradClipCallback'>"}, 'gpu_stats': {'every_n': '100', '_target_': "<class 'fastgen.callbacks.gpu_stats.GPUStatsCallback'>"}, 'train_profiler': {'every_n': '100', '_target_': "<class 'fastgen.callbacks.train_profiler.TrainProfilerCallback'>"}, 'param_count': {'_target_': "<class 'fastgen.callbacks.param_count.ParamCountCallback'>"}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': "<class 'fastgen.callbacks.ema.EMACallback'>"}, 'wandb': {'sample_logging_iter': '100', '_target_': "<class 'fastgen.callbacks.wandb.WandbCallback'>", 'fps': '25'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '5000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "<class 'fastgen.datasets.omniavatar_dataloader.OmniAvatarDataLoader'>"}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': '<function create_omniavatar_dataloader at 0x7f328019bce0>'}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "<class 'fastgen.networks.OmniAvatar.network_causal.CausalOmniAvatarWan'>"}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_phase2/step-10500.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "<class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>"}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt', 'merge_lora': 'False', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "<class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>"}, 'guidance_scale': '4.5', 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7f32803f2660>'}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.937', '0.833', '0.624', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '4', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'bfloat16', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7f32803f2660>'}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': "<class 'fastgen.networks.discriminators.Discriminator_EDM'>"}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7f32803f2660>'}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None}, 'model_class': {'config': None, '_target_': "<class 'fastgen.methods.omniavatar_self_forcing.OmniAvatarSelfForcingModel'>"}, '_wandb': {}}
+2026-04-03 00:01:32,170 INFO    MainThread:1047116 [wandb_init.py:init():892] starting backend
+2026-04-03 00:01:32,416 INFO    MainThread:1047116 [wandb_init.py:init():895] sending inform_init request
+2026-04-03 00:01:32,428 INFO    MainThread:1047116 [wandb_init.py:init():903] backend started and connected
+2026-04-03 00:01:32,432 INFO    MainThread:1047116 [wandb_init.py:init():973] updated telemetry
+2026-04-03 00:01:32,448 INFO    MainThread:1047116 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-04-03 00:01:33,668 INFO    MainThread:1047116 [wandb_init.py:init():1042] starting run threads in backend
+2026-04-03 00:01:33,982 INFO    MainThread:1047116 [wandb_run.py:_console_start():2524] atexit reg
+2026-04-03 00:01:33,982 INFO    MainThread:1047116 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-04-03 00:01:33,982 INFO    MainThread:1047116 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-04-03 00:01:33,982 INFO    MainThread:1047116 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-04-03 00:01:33,986 INFO    MainThread:1047116 [wandb_init.py:init():1082] run started, returning control to user process

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined/wandb/run-20260403_000132-nkf4iovm/files/output.log

@@ -0,0 +1,739 @@
+[Apr 3, 2026 - 00:01:33 | INFO | fastgen.callbacks.wandb:init_wandb:259 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable)
+[Apr 3, 2026 - 00:01:33 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully
+[Apr 3, 2026 - 00:01:33 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete
+[Apr 3, 2026 - 00:01:33 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer...
+[Apr 3, 2026 - 00:01:33 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully
+[Apr 3, 2026 - 00:01:33 | SUCCESS | __main__:main:33 ] Trainer initialized successfully
+[Apr 3, 2026 - 00:01:33 | INFO | fastgen.trainer:run:77 ] Starting training
+[Apr 3, 2026 - 00:01:33 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ...
+[Apr 3, 2026 - 00:01:33 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_lr1e5_10000iter_shift_5/checkpoints/0005000.pth
+[Apr 3, 2026 - 00:01:38 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict...
+[Apr 3, 2026 - 00:01:50 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: <All keys matched successfully>
+[Apr 3, 2026 - 00:01:50 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_lr1e5_10000iter_shift_5/checkpoints/0005000.pth at iteration 5000
+[Apr 3, 2026 - 00:01:50 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 5000.
+[Apr 3, 2026 - 00:01:50 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ...
+[Apr 3, 2026 - 00:01:50 | INFO | fastgen.methods.model:on_train_begin:296 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True
+[Apr 3, 2026 - 00:01:50 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed
+[Apr 3, 2026 - 00:01:50 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp ..
+[Apr 3, 2026 - 00:01:50 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks...
+[Apr 3, 2026 - 00:01:50 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)...
+[Apr 3, 2026 - 00:01:50 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:1950 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics)
+[Apr 3, 2026 - 00:01:50 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding
+[Apr 3, 2026 - 00:01:51 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 1.1s
+[Apr 3, 2026 - 00:01:51 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.60B params)...
+[Apr 3, 2026 - 00:01:51 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:765 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection)
+[Apr 3, 2026 - 00:01:52 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding
+[Apr 3, 2026 - 00:01:52 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 1.5s
+[Apr 3, 2026 - 00:01:52 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)...
+[Apr 3, 2026 - 00:01:52 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:765 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection)
+[Apr 3, 2026 - 00:02:03 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding
+[Apr 3, 2026 - 00:02:03 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 10.9s
+[Apr 3, 2026 - 00:02:03 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed
+[Apr 3, 2026 - 00:02:03 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback.
+[Apr 3, 2026 - 00:02:03 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None
+[Apr 3, 2026 - 00:02:03 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 0.
+[Apr 3, 2026 - 00:02:03 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader...
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:361 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingModel) has 1596.36 M trainable and 17311.83 M total params (logical).
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingModel) has 420.42 M trainable and 4350.43 M total params LOCAL on rank 0.
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingModel) has 420.14 M trainable and 4348.63 M total params LOCAL on rank 1.
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingModel) has 420.14 M trainable and 4348.63 M total params LOCAL on rank 2.
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingModel) has 420.14 M trainable and 4348.63 M total params LOCAL on rank 3.
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical).
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.68 M trainable and 376.68 M total params LOCAL on rank 0.
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 1.
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 2.
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 3.
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 174.98 M trainable and 1596.36 M total params (logical).
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.30 M total params LOCAL on rank 0.
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 1.
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 2.
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 3.
+[Apr 3, 2026 - 00:02:09 | INFO | fastgen.trainer:run:174 ] iter_start: 0
+[MEM] fake_score_update: START: alloc=9.45GB reserved=9.88GB peak=9.60GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=12.28GB reserved=49.39GB peak=45.74GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=13.16GB peak=45.74GB
+[MEM-fwd] after block 0: alloc=13.74GB peak=45.74GB
+[MEM-fwd] after block 10: alloc=19.55GB peak=45.74GB
+[MEM-fwd] after block 20: alloc=24.84GB peak=45.74GB
+[MEM-fwd] after block 29: alloc=29.59GB peak=45.74GB
+[MEM-fwd] after head+unpatchify: alloc=30.67GB peak=45.74GB
+[MEM] fake_score_update: START: alloc=13.25GB reserved=15.24GB peak=54.53GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=13.29GB reserved=47.38GB peak=46.75GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=14.17GB peak=46.75GB
+[MEM-fwd] after block 0: alloc=14.75GB peak=46.75GB
+[MEM-fwd] after block 10: alloc=20.56GB peak=46.75GB
+[MEM-fwd] after block 20: alloc=25.85GB peak=46.75GB
+[MEM-fwd] after block 29: alloc=30.61GB peak=46.75GB
+[MEM-fwd] after head+unpatchify: alloc=31.69GB peak=46.75GB
+[Apr 3, 2026 - 00:03:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg        Max        Min
+cpu_mem_gb                38.485269  38.565594  38.411797
+peak_gpu_mem_gb           51.700073  51.700073  51.700073
+peak_gpu_mem_reserved_gb  53.640625  53.640625  53.640625
+util                      89.250000  96.000000  84.000000
+[Apr 3, 2026 - 00:03:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 1 : data loading time        0.81
+[Apr 3, 2026 - 00:03:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 1 : avg forward pass time    15.10
+[Apr 3, 2026 - 00:03:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 1 : backward pass time       11.36
+[Apr 3, 2026 - 00:03:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 1 : optimizer step time      1.27
+[Apr 3, 2026 - 00:03:06 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 1--------------------
+[Apr 3, 2026 - 00:03:06 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.0068        iter count: 1.0
+[Apr 3, 2026 - 00:03:06 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0068   iter count: 1.0
+[Apr 3, 2026 - 00:03:06 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_disc: 0.0000     iter count: 1.0
+[Apr 3, 2026 - 00:03:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=13.43GB reserved=57.60GB peak=55.51GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=13.47GB reserved=47.47GB peak=46.93GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=14.35GB peak=46.93GB
+[MEM-fwd] after block 0: alloc=14.93GB peak=46.93GB
+[MEM-fwd] after block 10: alloc=20.74GB peak=46.93GB
+[MEM-fwd] after block 20: alloc=26.02GB peak=46.93GB
+[MEM-fwd] after block 29: alloc=30.78GB peak=46.93GB
+[MEM-fwd] after head+unpatchify: alloc=31.86GB peak=46.93GB
+[MEM] fake_score_update: START: alloc=14.21GB reserved=58.37GB peak=55.69GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=14.25GB reserved=48.25GB peak=47.71GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=15.13GB peak=47.71GB
+[MEM-fwd] after block 0: alloc=15.71GB peak=47.71GB
+[MEM-fwd] after block 10: alloc=21.52GB peak=47.71GB
+[MEM-fwd] after block 20: alloc=26.81GB peak=47.71GB
+[MEM-fwd] after block 29: alloc=31.57GB peak=47.71GB
+[MEM-fwd] after head+unpatchify: alloc=32.65GB peak=47.71GB
+[Apr 3, 2026 - 00:03:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg         Max        Min
+cpu_mem_gb                38.658718   38.739353  38.585140
+peak_gpu_mem_gb           52.593685   52.593685  52.593685
+peak_gpu_mem_reserved_gb  54.365234   54.365234  54.365234
+util                      95.750000  100.000000  92.000000
+[Apr 3, 2026 - 00:03:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 2 : avg iteration time       51.59 seconds
+[Apr 3, 2026 - 00:03:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 2 : data loading time        0.00
+[Apr 3, 2026 - 00:03:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 2 : avg forward pass time    13.82
+[Apr 3, 2026 - 00:03:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 2 : backward pass time       11.41
+[Apr 3, 2026 - 00:03:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 2 : optimizer step time      1.17
+[Apr 3, 2026 - 00:03:58 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 2--------------------
+[Apr 3, 2026 - 00:03:58 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.0070        iter count: 1.0
+[Apr 3, 2026 - 00:03:58 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0070   iter count: 1.0
+[Apr 3, 2026 - 00:03:58 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_disc: 0.0000     iter count: 1.0
+[Apr 3, 2026 - 00:03:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=13.43GB reserved=58.37GB peak=56.47GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=13.47GB reserved=47.49GB peak=46.93GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=14.35GB peak=46.93GB
+[MEM-fwd] after block 0: alloc=14.93GB peak=46.93GB
+[MEM-fwd] after block 10: alloc=20.74GB peak=46.93GB
+[MEM-fwd] after block 20: alloc=26.02GB peak=46.93GB
+[MEM-fwd] after block 29: alloc=30.78GB peak=46.93GB
+[MEM-fwd] after head+unpatchify: alloc=31.86GB peak=46.93GB
+[MEM] fake_score_update: START: alloc=14.21GB reserved=58.40GB peak=55.69GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=14.25GB reserved=48.29GB peak=47.72GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=15.13GB peak=47.72GB
+[MEM-fwd] after block 0: alloc=15.71GB peak=47.72GB
+[MEM-fwd] after block 10: alloc=21.52GB peak=47.72GB
+[MEM-fwd] after block 20: alloc=26.81GB peak=47.72GB
+[MEM-fwd] after block 29: alloc=31.57GB peak=47.72GB
+[MEM-fwd] after head+unpatchify: alloc=32.65GB peak=47.72GB
+[Apr 3, 2026 - 00:04:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg         Max        Min
+cpu_mem_gb                38.659063   38.740227  38.585308
+peak_gpu_mem_gb           52.593685   52.593685  52.593685
+peak_gpu_mem_reserved_gb  54.404297   54.404297  54.404297
+util                      96.750000  100.000000  92.000000
+[Apr 3, 2026 - 00:04:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 3 : avg iteration time       58.07 seconds
+[Apr 3, 2026 - 00:04:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 3 : data loading time        0.00
+[Apr 3, 2026 - 00:04:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 3 : avg forward pass time    17.04
+[Apr 3, 2026 - 00:04:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 3 : backward pass time       11.38
+[Apr 3, 2026 - 00:04:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 3 : optimizer step time      1.18
+[Apr 3, 2026 - 00:04:56 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 3--------------------
+[Apr 3, 2026 - 00:04:56 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.0072        iter count: 1.0
+[Apr 3, 2026 - 00:04:56 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0072   iter count: 1.0
+[Apr 3, 2026 - 00:04:56 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_disc: 0.0000     iter count: 1.0
+[Apr 3, 2026 - 00:04:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=13.43GB reserved=58.42GB peak=56.47GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=13.47GB reserved=47.52GB peak=46.93GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=14.35GB peak=46.93GB
+[MEM-fwd] after block 0: alloc=14.93GB peak=46.93GB
+[MEM-fwd] after block 10: alloc=20.74GB peak=46.93GB
+[MEM-fwd] after block 20: alloc=26.02GB peak=46.93GB
+[MEM-fwd] after block 29: alloc=30.78GB peak=46.93GB
+[MEM-fwd] after head+unpatchify: alloc=31.86GB peak=46.93GB
+[MEM] fake_score_update: START: alloc=14.21GB reserved=58.42GB peak=55.69GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=14.25GB reserved=48.27GB peak=47.72GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=15.13GB peak=47.72GB
+[MEM-fwd] after block 0: alloc=15.71GB peak=47.72GB
+[MEM-fwd] after block 10: alloc=21.52GB peak=47.72GB
+[MEM-fwd] after block 20: alloc=26.81GB peak=47.72GB
+[MEM-fwd] after block 29: alloc=31.57GB peak=47.72GB
+[MEM-fwd] after head+unpatchify: alloc=32.65GB peak=47.72GB
+[Apr 3, 2026 - 00:05:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg         Max        Min
+cpu_mem_gb                38.659183   38.740608  38.585339
+peak_gpu_mem_gb           52.593685   52.593685  52.593685
+peak_gpu_mem_reserved_gb  54.443359   54.443359  54.443359
+util                      95.250000  100.000000  91.000000
+[Apr 3, 2026 - 00:05:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 4 : avg iteration time       57.93 seconds
+[Apr 3, 2026 - 00:05:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 4 : data loading time        0.00
+[Apr 3, 2026 - 00:05:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 4 : avg forward pass time    17.00
+[Apr 3, 2026 - 00:05:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 4 : backward pass time       11.36
+[Apr 3, 2026 - 00:05:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 4 : optimizer step time      1.18
+[Apr 3, 2026 - 00:05:54 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 4--------------------
+[Apr 3, 2026 - 00:05:54 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.0073        iter count: 1.0
+[Apr 3, 2026 - 00:05:54 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0073   iter count: 1.0
+[Apr 3, 2026 - 00:05:54 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_disc: 0.0000     iter count: 1.0
+[Apr 3, 2026 - 00:05:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=13.43GB reserved=58.46GB peak=56.47GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=13.47GB reserved=47.49GB peak=46.93GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=14.35GB peak=46.93GB
+[MEM-fwd] after block 0: alloc=14.93GB peak=46.93GB
+[MEM-fwd] after block 10: alloc=20.74GB peak=46.93GB
+[MEM-fwd] after block 20: alloc=26.02GB peak=46.93GB
+[MEM-fwd] after block 29: alloc=30.78GB peak=46.93GB
+[MEM-fwd] after head+unpatchify: alloc=31.86GB peak=46.93GB
+[MEM] student_update: START: alloc=14.32GB reserved=58.46GB peak=55.75GB
+[MEM] student_update: after rollout: alloc=63.85GB reserved=66.24GB peak=65.62GB
+[MEM] student_update: after perturb: alloc=63.87GB reserved=66.24GB peak=65.62GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=64.62GB peak=65.62GB
+[MEM-fwd] after block 0: alloc=64.67GB peak=75.08GB
+[MEM-fwd] after block 10: alloc=65.20GB peak=75.61GB
+[MEM-fwd] after block 20: alloc=65.20GB peak=75.61GB
+[MEM-fwd] after block 29: alloc=65.20GB peak=75.61GB
+[MEM-fwd] after head+unpatchify: alloc=64.69GB peak=75.61GB
+[MEM] student_update: after fake_score: alloc=63.95GB reserved=78.02GB peak=75.61GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 5120]), 40 blocks): alloc=66.83GB peak=75.61GB
+[MEM-fwd] after block 0: alloc=67.05GB peak=88.90GB
+[MEM-fwd] after block 10: alloc=68.82GB peak=90.66GB
+[MEM-fwd] after block 20: alloc=68.82GB peak=90.66GB
+[MEM-fwd] after block 30: alloc=68.82GB peak=90.66GB
+[MEM-fwd] after block 39: alloc=68.82GB peak=90.66GB
+[MEM-fwd] after head+unpatchify: alloc=67.08GB peak=90.66GB
+[MEM] student_update: after teacher: alloc=64.45GB reserved=93.79GB peak=90.66GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 5120]), 40 blocks): alloc=66.37GB peak=90.66GB
+[MEM-fwd] after block 0: alloc=67.08GB peak=90.66GB
+[MEM-fwd] after block 10: alloc=68.84GB peak=90.68GB
+[MEM-fwd] after block 20: alloc=68.84GB peak=90.68GB
+[MEM-fwd] after block 30: alloc=68.84GB peak=90.68GB
+[MEM-fwd] after block 39: alloc=68.84GB peak=90.68GB
+[MEM-fwd] after head+unpatchify: alloc=67.10GB peak=90.68GB
+[MEM] student_update: after CFG: alloc=64.45GB reserved=93.83GB peak=90.68GB
+[MEM] student_update: after VSD loss: alloc=64.47GB reserved=93.83GB peak=90.68GB
+[MEM] fake_score_update: START: alloc=12.68GB reserved=72.00GB peak=90.68GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=15.51GB reserved=52.62GB peak=48.98GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=16.28GB peak=48.98GB
+[MEM-fwd] after block 0: alloc=16.91GB peak=48.98GB
+[MEM-fwd] after block 10: alloc=22.73GB peak=48.98GB
+[MEM-fwd] after block 20: alloc=28.01GB peak=48.98GB
+[MEM-fwd] after block 29: alloc=32.77GB peak=48.98GB
+[MEM-fwd] after head+unpatchify: alloc=33.85GB peak=48.98GB
+[MEM] student_update: START: alloc=15.52GB reserved=61.80GB peak=56.95GB
+[MEM] student_update: after rollout: alloc=65.05GB reserved=68.04GB peak=66.83GB
+[MEM] student_update: after perturb: alloc=65.07GB reserved=68.04GB peak=66.83GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=65.82GB peak=66.83GB
+[MEM-fwd] after block 0: alloc=65.87GB peak=76.28GB
+[MEM-fwd] after block 10: alloc=66.40GB peak=76.81GB
+[MEM-fwd] after block 20: alloc=66.40GB peak=76.81GB
+[MEM-fwd] after block 29: alloc=66.40GB peak=76.81GB
+[MEM-fwd] after head+unpatchify: alloc=65.90GB peak=76.81GB
+[MEM] student_update: after fake_score: alloc=65.15GB reserved=79.81GB peak=76.81GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 5120]), 40 blocks): alloc=67.08GB peak=76.81GB
+[MEM-fwd] after block 0: alloc=67.78GB peak=89.62GB
+[MEM-fwd] after block 10: alloc=69.54GB peak=91.39GB
+[MEM-fwd] after block 20: alloc=69.54GB peak=91.39GB
+[MEM-fwd] after block 30: alloc=69.54GB peak=91.39GB
+[MEM-fwd] after block 39: alloc=69.54GB peak=91.39GB
+[MEM-fwd] after head+unpatchify: alloc=67.80GB peak=91.39GB
+[MEM] student_update: after teacher: alloc=65.17GB reserved=94.89GB peak=91.39GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 5120]), 40 blocks): alloc=67.10GB peak=91.39GB
+[MEM-fwd] after block 0: alloc=67.80GB peak=91.39GB
+[MEM-fwd] after block 10: alloc=69.57GB peak=91.41GB
+[MEM-fwd] after block 20: alloc=69.57GB peak=91.41GB
+[MEM-fwd] after block 30: alloc=69.57GB peak=91.41GB
+[MEM-fwd] after block 39: alloc=69.57GB peak=91.41GB
+[MEM-fwd] after head+unpatchify: alloc=67.83GB peak=91.41GB
+[MEM] student_update: after CFG: alloc=65.17GB reserved=94.91GB peak=91.41GB
+[MEM] student_update: after VSD loss: alloc=65.20GB reserved=94.91GB peak=91.41GB
+[Apr 3, 2026 - 00:09:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg        Max        Min
+cpu_mem_gb                38.678225  38.757980  38.605282
+peak_gpu_mem_gb           85.130531  85.130531  85.130531
+peak_gpu_mem_reserved_gb  90.766602  90.771484  90.751953
+util                      67.500000  72.000000  61.000000
+[Apr 3, 2026 - 00:09:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 5 : avg iteration time       193.31 seconds
+[Apr 3, 2026 - 00:09:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 5 : data loading time        0.00
+[Apr 3, 2026 - 00:09:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 5 : avg forward pass time    86.26
+[Apr 3, 2026 - 00:09:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 5 : backward pass time       10.03
+[Apr 3, 2026 - 00:09:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 5 : optimizer step time      0.31
+[Apr 3, 2026 - 00:09:07 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 5--------------------
+[Apr 3, 2026 - 00:09:07 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.6670        iter count: 1.0
+[Apr 3, 2026 - 00:09:07 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_vsd_loss: 0.6670          iter count: 1.0
+[Apr 3, 2026 - 00:09:07 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_gen: 0.0000      iter count: 1.0
+[Apr 3, 2026 - 00:09:07 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0075   iter count: 1.0
+[Apr 3, 2026 - 00:09:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=13.43GB reserved=97.44GB peak=91.41GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=16.26GB reserved=53.38GB peak=49.73GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=17.03GB peak=49.73GB
+[MEM-fwd] after block 0: alloc=17.66GB peak=49.73GB
+[MEM-fwd] after block 10: alloc=23.48GB peak=49.73GB
+[MEM-fwd] after block 20: alloc=28.76GB peak=49.73GB
+[MEM-fwd] after block 29: alloc=33.52GB peak=49.73GB
+[MEM-fwd] after head+unpatchify: alloc=34.60GB peak=49.73GB
+[MEM] fake_score_update: START: alloc=16.19GB reserved=62.77GB peak=58.43GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=16.23GB reserved=50.28GB peak=49.69GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=17.11GB peak=49.69GB
+[MEM-fwd] after block 0: alloc=17.69GB peak=49.69GB
+[MEM-fwd] after block 10: alloc=23.50GB peak=49.69GB
+[MEM-fwd] after block 20: alloc=28.78GB peak=49.69GB
+[MEM-fwd] after block 29: alloc=33.54GB peak=49.69GB
+[MEM-fwd] after head+unpatchify: alloc=34.62GB peak=49.69GB
+[Apr 3, 2026 - 00:09:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg         Max        Min
+cpu_mem_gb                38.678251   38.758018  38.605293
+peak_gpu_mem_gb           54.432534   54.433307  54.432277
+peak_gpu_mem_reserved_gb  57.687500   58.478516  55.314453
+util                      98.500000  100.000000  96.000000
+[Apr 3, 2026 - 00:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 6 : avg iteration time       52.27 seconds
+[Apr 3, 2026 - 00:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 6 : data loading time        0.00
+[Apr 3, 2026 - 00:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 6 : avg forward pass time    14.22
+[Apr 3, 2026 - 00:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 6 : backward pass time       11.33
+[Apr 3, 2026 - 00:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 6 : optimizer step time      1.17
+[Apr 3, 2026 - 00:09:59 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 6--------------------
+[Apr 3, 2026 - 00:09:59 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.0073        iter count: 1.0
+[Apr 3, 2026 - 00:09:59 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0073   iter count: 1.0
+[Apr 3, 2026 - 00:09:59 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_disc: 0.0000     iter count: 1.0
+[Apr 3, 2026 - 00:09:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=15.40GB reserved=59.39GB peak=58.45GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=15.44GB reserved=49.46GB peak=48.91GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=16.32GB peak=48.91GB
+[MEM-fwd] after block 0: alloc=16.90GB peak=48.91GB
+[MEM-fwd] after block 10: alloc=22.71GB peak=48.91GB
+[MEM-fwd] after block 20: alloc=28.00GB peak=48.91GB
+[MEM-fwd] after block 29: alloc=32.76GB peak=48.91GB
+[MEM-fwd] after head+unpatchify: alloc=33.84GB peak=48.91GB
+[MEM] fake_score_update: START: alloc=16.19GB reserved=59.79GB peak=57.66GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=16.23GB reserved=50.28GB peak=49.69GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=17.11GB peak=49.69GB
+[MEM-fwd] after block 0: alloc=17.69GB peak=49.69GB
+[MEM-fwd] after block 10: alloc=23.50GB peak=49.69GB
+[MEM-fwd] after block 20: alloc=28.78GB peak=49.69GB
+[MEM-fwd] after block 29: alloc=33.54GB peak=49.69GB
+[MEM-fwd] after head+unpatchify: alloc=34.62GB peak=49.69GB
+[Apr 3, 2026 - 00:10:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg         Max        Min
+cpu_mem_gb                38.678332   38.758266  38.605301
+peak_gpu_mem_gb           54.432534   54.433307  54.432277
+peak_gpu_mem_reserved_gb  57.311523   57.853516  55.685547
+util                      97.250000  100.000000  92.000000
+[Apr 3, 2026 - 00:10:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 7 : avg iteration time       57.95 seconds
+[Apr 3, 2026 - 00:10:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 7 : data loading time        0.00
+[Apr 3, 2026 - 00:10:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 7 : avg forward pass time    17.05
+[Apr 3, 2026 - 00:10:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 7 : backward pass time       11.35
+[Apr 3, 2026 - 00:10:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 7 : optimizer step time      1.17
+[Apr 3, 2026 - 00:10:57 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 7--------------------
+[Apr 3, 2026 - 00:10:57 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.0071        iter count: 1.0
+[Apr 3, 2026 - 00:10:57 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0071   iter count: 1.0
+[Apr 3, 2026 - 00:10:57 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_disc: 0.0000     iter count: 1.0
+[Apr 3, 2026 - 00:10:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=15.40GB reserved=59.79GB peak=58.45GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=15.44GB reserved=49.50GB peak=48.91GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=16.32GB peak=48.91GB
+[MEM-fwd] after block 0: alloc=16.90GB peak=48.91GB
+[MEM-fwd] after block 10: alloc=22.71GB peak=48.91GB
+[MEM-fwd] after block 20: alloc=28.00GB peak=48.91GB
+[MEM-fwd] after block 29: alloc=32.76GB peak=48.91GB
+[MEM-fwd] after head+unpatchify: alloc=33.84GB peak=48.91GB
+[MEM] fake_score_update: START: alloc=16.19GB reserved=59.71GB peak=57.66GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=16.23GB reserved=50.28GB peak=49.69GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=17.11GB peak=49.69GB
+[MEM-fwd] after block 0: alloc=17.69GB peak=49.69GB
+[MEM-fwd] after block 10: alloc=23.50GB peak=49.69GB
+[MEM-fwd] after block 20: alloc=28.78GB peak=49.69GB
+[MEM-fwd] after block 29: alloc=33.54GB peak=49.69GB
+[MEM-fwd] after head+unpatchify: alloc=34.62GB peak=49.69GB
+[Apr 3, 2026 - 00:11:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg         Max        Min
+cpu_mem_gb                38.678370   38.758301  38.605301
+peak_gpu_mem_gb           54.432534   54.433307  54.432277
+peak_gpu_mem_reserved_gb  55.509766   55.685547  55.451172
+util                      97.500000  100.000000  95.000000
+[Apr 3, 2026 - 00:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 8 : avg iteration time       57.73 seconds
+[Apr 3, 2026 - 00:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 8 : data loading time        0.00
+[Apr 3, 2026 - 00:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 8 : avg forward pass time    16.95
+[Apr 3, 2026 - 00:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 8 : backward pass time       11.31
+[Apr 3, 2026 - 00:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 8 : optimizer step time      1.18
+[Apr 3, 2026 - 00:11:55 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 8--------------------
+[Apr 3, 2026 - 00:11:55 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.0067        iter count: 1.0
+[Apr 3, 2026 - 00:11:55 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0067   iter count: 1.0
+[Apr 3, 2026 - 00:11:55 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_disc: 0.0000     iter count: 1.0
+[Apr 3, 2026 - 00:11:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=15.40GB reserved=59.79GB peak=58.45GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=15.44GB reserved=49.48GB peak=48.91GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=16.32GB peak=48.91GB
+[MEM-fwd] after block 0: alloc=16.90GB peak=48.91GB
+[MEM-fwd] after block 10: alloc=22.71GB peak=48.91GB
+[MEM-fwd] after block 20: alloc=28.00GB peak=48.91GB
+[MEM-fwd] after block 29: alloc=32.76GB peak=48.91GB
+[MEM-fwd] after head+unpatchify: alloc=33.84GB peak=48.91GB
+[MEM] fake_score_update: START: alloc=16.19GB reserved=59.81GB peak=57.66GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=16.23GB reserved=50.30GB peak=49.69GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=17.11GB peak=49.69GB
+[MEM-fwd] after block 0: alloc=17.69GB peak=49.69GB
+[MEM-fwd] after block 10: alloc=23.50GB peak=49.69GB
+[MEM-fwd] after block 20: alloc=28.78GB peak=49.69GB
+[MEM-fwd] after block 29: alloc=33.54GB peak=49.69GB
+[MEM-fwd] after head+unpatchify: alloc=34.62GB peak=49.69GB
+[Apr 3, 2026 - 00:12:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg        Max        Min
+cpu_mem_gb                38.678534  38.758778  38.605354
+peak_gpu_mem_gb           54.432534  54.433307  54.432277
+peak_gpu_mem_reserved_gb  55.543945  55.705078  55.490234
+util                      96.750000  99.000000  94.000000
+[Apr 3, 2026 - 00:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 9 : avg iteration time       61.18 seconds
+[Apr 3, 2026 - 00:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 9 : data loading time        0.00
+[Apr 3, 2026 - 00:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 9 : avg forward pass time    18.67
+[Apr 3, 2026 - 00:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 9 : backward pass time       11.33
+[Apr 3, 2026 - 00:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 9 : optimizer step time      1.18
+[Apr 3, 2026 - 00:12:56 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 9--------------------
+[Apr 3, 2026 - 00:12:56 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.0071        iter count: 1.0
+[Apr 3, 2026 - 00:12:56 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0071   iter count: 1.0
+[Apr 3, 2026 - 00:12:56 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_disc: 0.0000     iter count: 1.0
+[Apr 3, 2026 - 00:12:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=15.40GB reserved=59.81GB peak=58.45GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=15.44GB reserved=49.50GB peak=48.91GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=16.32GB peak=48.91GB
+[MEM-fwd] after block 0: alloc=16.90GB peak=48.91GB
+[MEM-fwd] after block 10: alloc=22.71GB peak=48.91GB
+[MEM-fwd] after block 20: alloc=28.00GB peak=48.91GB
+[MEM-fwd] after block 29: alloc=32.76GB peak=48.91GB
+[MEM-fwd] after head+unpatchify: alloc=33.84GB peak=48.91GB
+[MEM] student_update: START: alloc=16.30GB reserved=59.83GB peak=57.72GB
+[MEM] student_update: after rollout: alloc=65.82GB reserved=68.23GB peak=67.60GB
+[MEM] student_update: after perturb: alloc=65.85GB reserved=68.23GB peak=67.60GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=66.60GB peak=67.60GB
+[MEM-fwd] after block 0: alloc=66.64GB peak=77.05GB
+[MEM-fwd] after block 10: alloc=67.17GB peak=77.58GB
+[MEM-fwd] after block 20: alloc=67.17GB peak=77.58GB
+[MEM-fwd] after block 29: alloc=67.17GB peak=77.58GB
+[MEM-fwd] after head+unpatchify: alloc=66.67GB peak=77.58GB
+[MEM] student_update: after fake_score: alloc=65.92GB reserved=79.54GB peak=77.58GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 5120]), 40 blocks): alloc=67.85GB peak=77.58GB
+[MEM-fwd] after block 0: alloc=68.55GB peak=90.40GB
+[MEM-fwd] after block 10: alloc=70.31GB peak=92.16GB
+[MEM-fwd] after block 20: alloc=70.31GB peak=92.16GB
+[MEM-fwd] after block 30: alloc=70.31GB peak=92.16GB
+[MEM-fwd] after block 39: alloc=70.31GB peak=92.16GB
+[MEM-fwd] after head+unpatchify: alloc=68.58GB peak=92.16GB
+[MEM] student_update: after teacher: alloc=65.95GB reserved=95.14GB peak=92.16GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 5120]), 40 blocks): alloc=67.87GB peak=92.16GB
+[MEM-fwd] after block 0: alloc=68.58GB peak=92.16GB
+[MEM-fwd] after block 10: alloc=70.34GB peak=92.18GB
+[MEM-fwd] after block 20: alloc=70.34GB peak=92.18GB
+[MEM-fwd] after block 30: alloc=70.34GB peak=92.18GB
+[MEM-fwd] after block 39: alloc=70.34GB peak=92.18GB
+[MEM-fwd] after head+unpatchify: alloc=68.60GB peak=92.18GB
+[MEM] student_update: after CFG: alloc=65.95GB reserved=95.16GB peak=92.18GB
+[MEM] student_update: after VSD loss: alloc=65.97GB reserved=95.16GB peak=92.18GB
+[MEM] fake_score_update: START: alloc=14.18GB reserved=98.71GB peak=92.18GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=17.01GB reserved=54.14GB peak=50.47GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=17.78GB peak=50.47GB
+[MEM-fwd] after block 0: alloc=18.41GB peak=50.47GB
+[MEM-fwd] after block 10: alloc=24.23GB peak=50.47GB
+[MEM-fwd] after block 20: alloc=29.51GB peak=50.47GB
+[MEM-fwd] after block 29: alloc=34.27GB peak=50.47GB
+[MEM-fwd] after head+unpatchify: alloc=35.35GB peak=50.47GB
+[MEM] student_update: START: alloc=17.02GB reserved=66.65GB peak=58.45GB
+[MEM] student_update: after rollout: alloc=66.55GB reserved=69.49GB peak=68.33GB
+[MEM] student_update: after perturb: alloc=66.57GB reserved=69.49GB peak=68.33GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=67.32GB peak=68.33GB
+[MEM-fwd] after block 0: alloc=67.37GB peak=77.78GB
+[MEM-fwd] after block 10: alloc=67.90GB peak=78.31GB
+[MEM-fwd] after block 20: alloc=67.90GB peak=78.31GB
+[MEM-fwd] after block 29: alloc=67.90GB peak=78.31GB
+[MEM-fwd] after head+unpatchify: alloc=67.39GB peak=78.31GB
+[MEM] student_update: after fake_score: alloc=66.65GB reserved=81.28GB peak=78.31GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 5120]), 40 blocks): alloc=68.58GB peak=78.31GB
+[MEM-fwd] after block 0: alloc=69.28GB peak=91.12GB
+[MEM-fwd] after block 10: alloc=71.04GB peak=92.88GB
+[MEM-fwd] after block 20: alloc=71.04GB peak=92.88GB
+[MEM-fwd] after block 30: alloc=71.04GB peak=92.88GB
+[MEM-fwd] after block 39: alloc=71.04GB peak=92.88GB
+[MEM-fwd] after head+unpatchify: alloc=69.30GB peak=92.88GB
+[MEM] student_update: after teacher: alloc=66.67GB reserved=96.36GB peak=92.88GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 5120]), 40 blocks): alloc=68.60GB peak=92.88GB
+[MEM-fwd] after block 0: alloc=69.30GB peak=92.88GB
+[MEM-fwd] after block 10: alloc=71.06GB peak=92.91GB
+[MEM-fwd] after block 20: alloc=71.06GB peak=92.91GB
+[MEM-fwd] after block 30: alloc=71.06GB peak=92.91GB
+[MEM-fwd] after block 39: alloc=71.06GB peak=92.91GB
+[MEM-fwd] after head+unpatchify: alloc=69.32GB peak=92.91GB
+[MEM] student_update: after CFG: alloc=66.67GB reserved=96.36GB peak=92.91GB
+[MEM] student_update: after VSD loss: alloc=66.70GB reserved=96.36GB peak=92.91GB
+[Apr 3, 2026 - 00:16:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg        Max        Min
+cpu_mem_gb                38.679560  38.760101  38.606266
+peak_gpu_mem_gb           86.525435  86.526208  86.525178
+peak_gpu_mem_reserved_gb  92.133301  92.142578  92.105469
+util                      78.750000  83.000000  73.000000
+[Apr 3, 2026 - 00:16:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 10 : avg iteration time       185.69 seconds
+[Apr 3, 2026 - 00:16:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 10 : data loading time        0.00
+[Apr 3, 2026 - 00:16:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 10 : avg forward pass time    82.75
+[Apr 3, 2026 - 00:16:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 10 : backward pass time       10.03
+[Apr 3, 2026 - 00:16:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 10 : optimizer step time      0.21
+[Apr 3, 2026 - 00:16:02 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 10--------------------
+[Apr 3, 2026 - 00:16:02 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.7031        iter count: 1.0
+[Apr 3, 2026 - 00:16:02 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_vsd_loss: 0.7031          iter count: 1.0
+[Apr 3, 2026 - 00:16:02 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_gen: 0.0000      iter count: 1.0
+[Apr 3, 2026 - 00:16:02 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0071   iter count: 1.0
+[Apr 3, 2026 - 00:16:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=13.43GB reserved=98.90GB peak=92.91GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=16.26GB reserved=53.40GB peak=49.73GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=17.03GB peak=49.73GB
+[MEM-fwd] after block 0: alloc=17.66GB peak=49.73GB
+[MEM-fwd] after block 10: alloc=23.48GB peak=49.73GB
+[MEM-fwd] after block 20: alloc=28.76GB peak=49.73GB
+[MEM-fwd] after block 29: alloc=33.52GB peak=49.73GB
+[MEM-fwd] after head+unpatchify: alloc=34.60GB peak=49.73GB
+[MEM] fake_score_update: START: alloc=16.19GB reserved=62.56GB peak=58.43GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=16.23GB reserved=50.36GB peak=49.69GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=17.11GB peak=49.69GB
+[MEM-fwd] after block 0: alloc=17.69GB peak=49.69GB
+[MEM-fwd] after block 10: alloc=23.50GB peak=49.69GB
+[MEM-fwd] after block 20: alloc=28.78GB peak=49.69GB
+[MEM-fwd] after block 29: alloc=33.54GB peak=49.69GB
+[MEM-fwd] after head+unpatchify: alloc=34.62GB peak=49.69GB
+[Apr 3, 2026 - 00:17:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg         Max        Min
+cpu_mem_gb                38.679598   38.760120  38.606396
+peak_gpu_mem_gb           54.432534   54.433307  54.432277
+peak_gpu_mem_reserved_gb  55.583008   55.802734  55.509766
+util                      97.250000  100.000000  91.000000
+[Apr 3, 2026 - 00:17:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 11 : avg iteration time       58.86 seconds
+[Apr 3, 2026 - 00:17:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 11 : data loading time        0.00
+[Apr 3, 2026 - 00:17:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 11 : avg forward pass time    17.51
+[Apr 3, 2026 - 00:17:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 11 : backward pass time       11.32
+[Apr 3, 2026 - 00:17:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 11 : optimizer step time      1.20
+[Apr 3, 2026 - 00:17:01 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 11--------------------
+[Apr 3, 2026 - 00:17:01 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.0087        iter count: 1.0
+[Apr 3, 2026 - 00:17:01 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0087   iter count: 1.0
+[Apr 3, 2026 - 00:17:01 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_disc: 0.0000     iter count: 1.0
+[Apr 3, 2026 - 00:17:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=15.40GB reserved=59.92GB peak=58.45GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=15.44GB reserved=49.52GB peak=48.91GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=16.32GB peak=48.91GB
+[MEM-fwd] after block 0: alloc=16.90GB peak=48.91GB
+[MEM-fwd] after block 10: alloc=22.71GB peak=48.91GB
+[MEM-fwd] after block 20: alloc=28.00GB peak=48.91GB
+[MEM-fwd] after block 29: alloc=32.76GB peak=48.91GB
+[MEM-fwd] after head+unpatchify: alloc=33.84GB peak=48.91GB
+[MEM] fake_score_update: START: alloc=16.19GB reserved=59.44GB peak=57.66GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=16.23GB reserved=50.30GB peak=49.69GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=17.11GB peak=49.69GB
+[MEM-fwd] after block 0: alloc=17.69GB peak=49.69GB
+[MEM-fwd] after block 10: alloc=23.50GB peak=49.69GB
+[MEM-fwd] after block 20: alloc=28.78GB peak=49.69GB
+[MEM-fwd] after block 29: alloc=33.54GB peak=49.69GB
+[MEM-fwd] after head+unpatchify: alloc=34.62GB peak=49.69GB
+[Apr 3, 2026 - 00:17:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg         Max        Min
+cpu_mem_gb                38.679661   38.760372  38.606403
+peak_gpu_mem_gb           54.432534   54.433307  54.432277
+peak_gpu_mem_reserved_gb  55.514648   55.568359  55.353516
+util                      97.250000  100.000000  93.000000
+[Apr 3, 2026 - 00:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 12 : avg iteration time       54.55 seconds
+[Apr 3, 2026 - 00:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 12 : data loading time        0.00
+[Apr 3, 2026 - 00:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 12 : avg forward pass time    15.35
+[Apr 3, 2026 - 00:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 12 : backward pass time       11.34
+[Apr 3, 2026 - 00:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 12 : optimizer step time      1.17
+[Apr 3, 2026 - 00:17:55 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 12--------------------
+[Apr 3, 2026 - 00:17:55 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.0070        iter count: 1.0
+[Apr 3, 2026 - 00:17:55 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0070   iter count: 1.0
+[Apr 3, 2026 - 00:17:55 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_disc: 0.0000     iter count: 1.0
+[Apr 3, 2026 - 00:17:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=15.40GB reserved=59.44GB peak=58.45GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=15.44GB reserved=49.52GB peak=48.90GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=16.32GB peak=48.90GB
+[MEM-fwd] after block 0: alloc=16.90GB peak=48.90GB
+[MEM-fwd] after block 10: alloc=22.71GB peak=48.90GB
+[MEM-fwd] after block 20: alloc=28.00GB peak=48.90GB
+[MEM-fwd] after block 29: alloc=32.76GB peak=48.90GB
+[MEM-fwd] after head+unpatchify: alloc=33.84GB peak=48.90GB
+[MEM] fake_score_update: START: alloc=16.19GB reserved=59.46GB peak=57.66GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=16.23GB reserved=50.02GB peak=49.69GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=17.11GB peak=49.69GB
+[MEM-fwd] after block 0: alloc=17.69GB peak=49.69GB
+[MEM-fwd] after block 10: alloc=23.50GB peak=49.69GB
+[MEM-fwd] after block 20: alloc=28.78GB peak=49.69GB
+[MEM-fwd] after block 29: alloc=33.54GB peak=49.69GB
+[MEM-fwd] after head+unpatchify: alloc=34.62GB peak=49.69GB
+[Apr 3, 2026 - 00:18:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg        Max        Min
+cpu_mem_gb                38.679665  38.760372  38.606419
+peak_gpu_mem_gb           54.432534  54.433307  54.432277
+peak_gpu_mem_reserved_gb  57.541016  58.224609  55.490234
+util                      97.750000  99.000000  94.000000
+[Apr 3, 2026 - 00:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 13 : avg iteration time       48.16 seconds
+[Apr 3, 2026 - 00:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 13 : data loading time        0.00
+[Apr 3, 2026 - 00:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 13 : avg forward pass time    12.17
+[Apr 3, 2026 - 00:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 13 : backward pass time       11.32
+[Apr 3, 2026 - 00:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 13 : optimizer step time      1.17
+[Apr 3, 2026 - 00:18:43 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 13--------------------
+[Apr 3, 2026 - 00:18:43 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.0072        iter count: 1.0
+[Apr 3, 2026 - 00:18:43 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0072   iter count: 1.0
+[Apr 3, 2026 - 00:18:43 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_disc: 0.0000     iter count: 1.0
+[Apr 3, 2026 - 00:18:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=15.40GB reserved=59.58GB peak=58.45GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=15.44GB reserved=49.54GB peak=48.91GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=16.32GB peak=48.91GB
+[MEM-fwd] after block 0: alloc=16.90GB peak=48.91GB
+[MEM-fwd] after block 10: alloc=22.71GB peak=48.91GB
+[MEM-fwd] after block 20: alloc=28.00GB peak=48.91GB
+[MEM-fwd] after block 29: alloc=32.76GB peak=48.91GB
+[MEM-fwd] after head+unpatchify: alloc=33.84GB peak=48.91GB
+[MEM] fake_score_update: START: alloc=16.19GB reserved=59.44GB peak=57.66GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=16.23GB reserved=50.32GB peak=49.69GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=17.11GB peak=49.69GB
+[MEM-fwd] after block 0: alloc=17.69GB peak=49.69GB
+[MEM-fwd] after block 10: alloc=23.50GB peak=49.69GB
+[MEM-fwd] after block 20: alloc=28.78GB peak=49.69GB
+[MEM-fwd] after block 29: alloc=33.54GB peak=49.69GB
+[MEM-fwd] after head+unpatchify: alloc=34.62GB peak=49.69GB
+[Apr 3, 2026 - 00:19:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg         Max        Min
+cpu_mem_gb                38.679850   38.761127  38.606380
+peak_gpu_mem_gb           54.432534   54.433307  54.432277
+peak_gpu_mem_reserved_gb  55.739258   55.744141  55.724609
+util                      98.000000  100.000000  94.000000
+[Apr 3, 2026 - 00:19:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 14 : avg iteration time       57.85 seconds
+[Apr 3, 2026 - 00:19:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 14 : data loading time        0.00
+[Apr 3, 2026 - 00:19:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 14 : avg forward pass time    17.00
+[Apr 3, 2026 - 00:19:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 14 : backward pass time       11.34
+[Apr 3, 2026 - 00:19:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 14 : optimizer step time      1.18
+[Apr 3, 2026 - 00:19:41 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 14--------------------
+[Apr 3, 2026 - 00:19:41 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.0077        iter count: 1.0
+[Apr 3, 2026 - 00:19:41 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0077   iter count: 1.0
+[Apr 3, 2026 - 00:19:41 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_disc: 0.0000     iter count: 1.0
+[Apr 3, 2026 - 00:19:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=15.40GB reserved=59.83GB peak=58.45GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=15.44GB reserved=49.50GB peak=48.91GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=16.32GB peak=48.91GB
+[MEM-fwd] after block 0: alloc=16.90GB peak=48.91GB
+[MEM-fwd] after block 10: alloc=22.71GB peak=48.91GB
+[MEM-fwd] after block 20: alloc=28.00GB peak=48.91GB
+[MEM-fwd] after block 29: alloc=32.76GB peak=48.91GB
+[MEM-fwd] after head+unpatchify: alloc=33.84GB peak=48.91GB
+[MEM] student_update: START: alloc=16.30GB reserved=59.44GB peak=57.72GB
+[MEM] student_update: after rollout: alloc=65.84GB reserved=68.21GB peak=67.61GB
+[MEM] student_update: after perturb: alloc=65.86GB reserved=68.21GB peak=67.61GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=66.61GB peak=67.61GB
+[MEM-fwd] after block 0: alloc=66.66GB peak=77.07GB
+[MEM-fwd] after block 10: alloc=67.19GB peak=77.60GB
+[MEM-fwd] after block 20: alloc=67.19GB peak=77.60GB
+[MEM-fwd] after block 29: alloc=67.19GB peak=77.60GB
+[MEM-fwd] after head+unpatchify: alloc=66.68GB peak=77.60GB
+[MEM] student_update: after fake_score: alloc=65.94GB reserved=80.02GB peak=77.60GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 5120]), 40 blocks): alloc=67.86GB peak=77.60GB
+[MEM-fwd] after block 0: alloc=68.57GB peak=90.41GB
+[MEM-fwd] after block 10: alloc=70.33GB peak=92.17GB
+[MEM-fwd] after block 20: alloc=70.33GB peak=92.17GB
+[MEM-fwd] after block 30: alloc=70.33GB peak=92.17GB
+[MEM-fwd] after block 39: alloc=70.33GB peak=92.17GB
+[MEM-fwd] after head+unpatchify: alloc=68.59GB peak=92.17GB
+[MEM] student_update: after teacher: alloc=65.96GB reserved=95.12GB peak=92.17GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 5120]), 40 blocks): alloc=67.89GB peak=92.17GB
+[MEM-fwd] after block 0: alloc=68.59GB peak=92.17GB
+[MEM-fwd] after block 10: alloc=70.35GB peak=92.19GB
+[MEM-fwd] after block 20: alloc=70.35GB peak=92.19GB
+[MEM-fwd] after block 30: alloc=70.35GB peak=92.19GB
+[MEM-fwd] after block 39: alloc=70.35GB peak=92.19GB
+[MEM-fwd] after head+unpatchify: alloc=68.61GB peak=92.19GB
+[MEM] student_update: after CFG: alloc=65.96GB reserved=95.12GB peak=92.19GB
+[MEM] student_update: after VSD loss: alloc=65.98GB reserved=95.12GB peak=92.19GB
+[MEM] fake_score_update: START: alloc=14.18GB reserved=98.66GB peak=92.19GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=17.02GB reserved=54.16GB peak=50.48GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=17.78GB peak=50.48GB
+[MEM-fwd] after block 0: alloc=18.42GB peak=50.48GB
+[MEM-fwd] after block 10: alloc=24.23GB peak=50.48GB
+[MEM-fwd] after block 20: alloc=29.52GB peak=50.48GB
+[MEM-fwd] after block 29: alloc=34.27GB peak=50.48GB
+[MEM-fwd] after head+unpatchify: alloc=35.35GB peak=50.48GB
+[MEM] student_update: START: alloc=17.03GB reserved=63.28GB peak=58.45GB
+[MEM] student_update: after rollout: alloc=66.57GB reserved=69.53GB peak=68.34GB
+[MEM] student_update: after perturb: alloc=66.59GB reserved=69.53GB peak=68.34GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=67.34GB peak=68.34GB
+[MEM-fwd] after block 0: alloc=67.39GB peak=77.80GB
+[MEM-fwd] after block 10: alloc=67.92GB peak=78.33GB
+[MEM-fwd] after block 20: alloc=67.92GB peak=78.33GB
+[MEM-fwd] after block 29: alloc=67.92GB peak=78.33GB
+[MEM-fwd] after head+unpatchify: alloc=67.41GB peak=78.33GB
+[MEM] student_update: after fake_score: alloc=66.67GB reserved=80.78GB peak=78.33GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 5120]), 40 blocks): alloc=68.60GB peak=78.33GB
+[MEM-fwd] after block 0: alloc=69.30GB peak=91.14GB
+[MEM-fwd] after block 10: alloc=71.06GB peak=92.90GB
+[MEM-fwd] after block 20: alloc=71.06GB peak=92.90GB
+[MEM-fwd] after block 30: alloc=71.06GB peak=92.90GB
+[MEM-fwd] after block 39: alloc=71.06GB peak=92.90GB
+[MEM-fwd] after head+unpatchify: alloc=69.32GB peak=92.90GB
+[MEM] student_update: after teacher: alloc=66.69GB reserved=96.38GB peak=92.90GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 5120]), 40 blocks): alloc=68.62GB peak=92.90GB
+[MEM-fwd] after block 0: alloc=69.32GB peak=92.90GB
+[MEM-fwd] after block 10: alloc=71.08GB peak=92.93GB
+[MEM-fwd] after block 20: alloc=71.08GB peak=92.93GB
+[MEM-fwd] after block 30: alloc=71.08GB peak=92.93GB
+[MEM-fwd] after block 39: alloc=71.08GB peak=92.93GB
+[MEM-fwd] after head+unpatchify: alloc=69.34GB peak=92.93GB
+[MEM] student_update: after CFG: alloc=66.69GB reserved=96.42GB peak=92.93GB
+[MEM] student_update: after VSD loss: alloc=66.71GB reserved=96.42GB peak=92.93GB
+[Apr 3, 2026 - 00:22:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg        Max        Min
+cpu_mem_gb                38.679507  38.759026  38.606628
+peak_gpu_mem_gb           86.542802  86.543575  86.542545
+peak_gpu_mem_reserved_gb  92.208496  92.222656  92.166016
+util                      77.500000  82.000000  71.000000
+[Apr 3, 2026 - 00:22:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 15 : avg iteration time       182.57 seconds
+[Apr 3, 2026 - 00:22:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 15 : data loading time        0.00
+[Apr 3, 2026 - 00:22:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 15 : avg forward pass time    81.19
+[Apr 3, 2026 - 00:22:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 15 : backward pass time       10.03
+[Apr 3, 2026 - 00:22:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 15 : optimizer step time      0.22
+[Apr 3, 2026 - 00:22:44 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 15--------------------
+[Apr 3, 2026 - 00:22:44 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.6475        iter count: 1.0
+[Apr 3, 2026 - 00:22:44 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_vsd_loss: 0.6475          iter count: 1.0
+[Apr 3, 2026 - 00:22:44 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_gen: 0.0000      iter count: 1.0
+[Apr 3, 2026 - 00:22:44 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0067   iter count: 1.0
+[Apr 3, 2026 - 00:22:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=13.44GB reserved=98.96GB peak=92.93GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=16.27GB reserved=53.40GB peak=49.73GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=17.04GB peak=49.73GB
+[MEM-fwd] after block 0: alloc=17.67GB peak=49.73GB
+[MEM-fwd] after block 10: alloc=23.49GB peak=49.73GB
+[MEM-fwd] after block 20: alloc=28.77GB peak=49.73GB
+[MEM-fwd] after block 29: alloc=33.53GB peak=49.73GB
+[MEM-fwd] after head+unpatchify: alloc=34.61GB peak=49.73GB
+[MEM] fake_score_update: START: alloc=16.19GB reserved=62.56GB peak=58.43GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=16.24GB reserved=50.28GB peak=49.70GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=17.12GB peak=49.70GB
+[MEM-fwd] after block 0: alloc=17.69GB peak=49.70GB
+[MEM-fwd] after block 10: alloc=23.51GB peak=49.70GB
+[MEM-fwd] after block 20: alloc=28.79GB peak=49.70GB
+[MEM-fwd] after block 29: alloc=33.55GB peak=49.70GB
+[MEM-fwd] after head+unpatchify: alloc=34.63GB peak=49.70GB
+[Apr 3, 2026 - 00:23:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg         Max        Min
+cpu_mem_gb                38.679513   38.759029  38.606636
+peak_gpu_mem_gb           54.440308   54.441080  54.440050
+peak_gpu_mem_reserved_gb  57.801758   58.539062  55.589844
+util                      98.500000  100.000000  95.000000
+[Apr 3, 2026 - 00:23:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 16 : avg iteration time       58.84 seconds
+[Apr 3, 2026 - 00:23:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 16 : data loading time        0.00
+[Apr 3, 2026 - 00:23:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 16 : avg forward pass time    17.46
+[Apr 3, 2026 - 00:23:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 16 : backward pass time       11.37
+[Apr 3, 2026 - 00:23:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 16 : optimizer step time      1.18
+[Apr 3, 2026 - 00:23:43 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 16--------------------
+[Apr 3, 2026 - 00:23:43 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.0066        iter count: 1.0
+[Apr 3, 2026 - 00:23:43 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0066   iter count: 1.0
+[Apr 3, 2026 - 00:23:43 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_disc: 0.0000     iter count: 1.0
+[Apr 3, 2026 - 00:23:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=15.41GB reserved=59.69GB peak=58.46GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=15.45GB reserved=49.55GB peak=48.92GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=16.33GB peak=48.92GB
+[MEM-fwd] after block 0: alloc=16.91GB peak=48.92GB
+[MEM-fwd] after block 10: alloc=22.72GB peak=48.92GB
+[MEM-fwd] after block 20: alloc=28.01GB peak=48.92GB
+[MEM-fwd] after block 29: alloc=32.76GB peak=48.92GB
+[MEM-fwd] after head+unpatchify: alloc=33.84GB peak=48.92GB
+[MEM] fake_score_update: START: alloc=16.19GB reserved=59.46GB peak=57.67GB

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined/wandb/run-20260403_000132-nkf4iovm/files/requirements.txt

@@ -0,0 +1,222 @@
+fastgen==0.1.0
+nvitop==1.6.1
+ftfy==6.3.1
+braceexpand==0.1.7
+antlr4-python3-runtime==4.9.3
+webdataset==1.0.2
+sentry-sdk==2.53.0
+rdkit==2025.9.5
+python-dotenv==1.2.1
+proglog==0.1.12
+omegaconf==2.3.0
+narwhals==2.17.0
+loguru==0.7.3
+imageio-ffmpeg==0.6.0
+plotly==6.5.2
+moviepy==2.2.1
+hydra-core==1.3.2
+wandb==0.25.0
+fastgen==0.1.0
+packaging==25.0
+setuptools==80.10.2
+wheel==0.46.3
+pip==26.0.1
+webencodings==0.5.1
+pure_eval==0.2.3
+ptyprocess==0.7.0
+nvidia-ml-py==13.590.48
+nvidia-cusparselt-cu12==0.7.1
+mpmath==1.3.0
+fastjsonschema==2.21.2
+zipp==3.23.0
+xyzservices==2025.11.0
+widgetsnbextension==4.0.15
+websocket-client==1.9.0
+webcolors==25.10.0
+wcwidth==0.6.0
+urllib3==2.6.3
+uri-template==1.3.0
+tzdata==2025.3
+typing_extensions==4.15.0
+triton==3.6.0
+traitlets==5.14.3
+tqdm==4.67.3
+tornado==6.5.5
+tinycss2==1.4.0
+sympy==1.14.0
+soupsieve==2.8.3
+smmap==5.0.3
+six==1.16.0
+sentencepiece==0.2.1
+Send2Trash==2.1.0
+safetensors==0.7.0
+rpds-py==0.30.0
+rfc3986-validator==0.1.1
+regex==2026.2.28
+pyzmq==27.1.0
+PyYAML==6.0.3
+python-json-logger==4.0.0
+Pygments==2.19.2
+pycparser==3.0
+psutil==7.2.2
+protobuf==4.24.4
+prometheus_client==0.24.1
+platformdirs==4.9.4
+pillow==11.3.0
+pexpect==4.9.0
+parso==0.8.6
+pandocfilters==1.5.1
+nvidia-nvtx-cu12==12.8.90
+nvidia-nvshmem-cu12==3.4.5
+nvidia-nvjitlink-cu12==12.8.93
+nvidia-nccl-cu12==2.27.5
+nvidia-curand-cu12==10.3.9.90
+nvidia-cufile-cu12==1.13.1.3
+nvidia-cuda-runtime-cu12==12.8.90
+nvidia-cuda-nvrtc-cu12==12.8.93
+nvidia-cuda-cupti-cu12==12.8.90
+nvidia-cublas-cu12==12.8.4.1
+numpy==1.26.4
+networkx==3.6.1
+nest-asyncio==1.6.0
+mistune==3.2.0
+MarkupSafe==3.0.3
+lark==1.3.1
+jupyterlab_widgets==3.0.16
+jupyterlab_pygments==0.3.0
+jsonpointer==3.0.0
+json5==0.13.0
+jmespath==1.1.0
+idna==3.11
+hf-xet==1.4.2
+h11==0.16.0
+fsspec==2026.2.0
+fqdn==1.5.1
+filelock==3.25.2
+executing==2.2.1
+einops==0.8.2
+defusedxml==0.7.1
+decorator==5.2.1
+debugpy==1.8.20
+cuda-pathfinder==1.4.2
+comm==0.2.3
+click==8.3.1
+charset-normalizer==3.4.5
+certifi==2026.2.25
+bleach==6.3.0
+babel==2.18.0
+av==17.0.0
+attrs==25.4.0
+async-lru==2.2.0
+asttokens==3.0.1
+annotated-types==0.7.0
+typing-inspection==0.4.2
+terminado==0.18.1
+stack-data==0.6.3
+scipy==1.17.1
+rfc3987-syntax==1.1.0
+rfc3339-validator==0.1.4
+requests==2.32.5
+referencing==0.37.0
+python-dateutil==2.9.0.post0
+pydantic_core==2.41.5
+prompt_toolkit==3.0.52
+opencv-python-headless==4.11.0.86
+nvidia-cusparse-cu12==12.5.8.93
+nvidia-cufft-cu12==11.3.3.83
+nvidia-cudnn-cu12==9.10.2.21
+matplotlib-inline==0.2.1
+jupyter_core==5.9.1
+Jinja2==3.1.6
+jedi==0.19.2
+ipython_pygments_lexers==1.1.1
+importlib_metadata==8.7.1
+ImageIO==2.37.3
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==12.9.4
+contourpy==1.3.3
+cffi==2.0.0
+beautifulsoup4==4.14.3
+anyio==4.12.1
+soundfile==0.13.1
+pydantic==2.12.5
+nvidia-cusolver-cu12==11.7.3.90
+jupyter_server_terminals==0.5.4
+jupyter_client==8.8.0
+jsonschema-specifications==2025.9.1
+ipython==9.11.0
+httpx==0.28.1
+GitPython==3.1.46
+botocore==1.42.68
+bokeh==3.9.0
+arrow==1.4.0
+argon2-cffi-bindings==25.1.0
+torch==2.10.0
+s3transfer==0.16.0
+jsonschema==4.26.0
+isoduration==20.11.0
+ipywidgets==8.1.8
+ipykernel==7.2.0
+argon2-cffi==25.1.0
+torchvision==0.25.0
+nbformat==5.10.4
+jupyter-console==6.6.3
+boto3==1.42.68
+accelerate==1.13.0
+nbclient==0.10.4
+jupyter-events==0.12.0
+nbconvert==7.17.0
+jupyter_server==2.17.0
+notebook_shim==0.2.4
+jupyterlab_server==2.28.0
+jupyter-lsp==2.3.0
+jupyterlab==4.5.6
+notebook==7.5.5
+jupyter==1.1.1
+fastgen==0.1.0
+pandas==3.0.1
+shellingham==1.5.4
+mdurl==0.1.2
+annotated-doc==0.0.4
+markdown-it-py==4.0.0
+rich==14.3.3
+typer==0.24.1
+huggingface_hub==1.7.1
+timm==1.0.25
+tokenizers==0.22.2
+diffusers==0.37.0
+transformers==5.3.0
+peft==0.18.1
+easydict==1.13
+lmdb==2.2.0
+threadpoolctl==3.6.0
+soxr==1.0.0
+msgpack==1.1.2
+llvmlite==0.47.0
+lazy-loader==0.5
+joblib==1.5.3
+audioread==3.1.0
+scikit-learn==1.8.0
+pooch==1.9.0
+numba==0.65.0
+librosa==0.11.0
+simsimd==6.5.16
+flatbuffers==25.12.19
+tifffile==2026.3.3
+stringzilla==4.6.0
+pyparsing==3.3.2
+prettytable==3.17.0
+onnx==1.17.0
+kiwisolver==1.5.0
+fonttools==4.62.1
+Cython==3.2.4
+cycler==0.12.1
+scikit-image==0.26.0
+onnxruntime==1.24.4
+matplotlib==3.10.8
+albucore==0.0.24
+albumentations==2.0.8
+insightface==0.7.3
+kornia_rs==0.1.10
+kornia==0.8.2

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined/wandb/run-20260403_000132-nkf4iovm/files/wandb-metadata.json

@@ -0,0 +1,68 @@
+{
+  "os":  "Linux-5.15.0-151-generic-x86_64-with-glibc2.39",
+  "python":  "CPython 3.12.12",
+  "startedAt":  "2026-04-02T15:01:32.168210Z",
+  "args":  [
+    "--config=fastgen/configs/experiments/OmniAvatar/config_sf.py",
+    "-",
+    "trainer.resume=False",
+    "log_config.name=sf_4gpu_bs8_lr2e6_5000iter_shift5_combined",
+    "log_config.project=OmniAvatar-FastGen"
+  ],
+  "program":  "/home/work/.local/hyunbin/FastGen/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://paulcho98:@github.com/paulcho98/FastGen.git",
+    "commit":  "04de80beaf50f849c12a55a5d8358d94530b7bb5"
+  },
+  "email":  "paul.hyunbin@gmail.com",
+  "root":  "/tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined",
+  "host":  "main1",
+  "executable":  "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12",
+  "cpu_count":  112,
+  "cpu_count_logical":  224,
+  "gpu":  "NVIDIA H200",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "1356758433792",
+      "used":  "270456766464"
+    }
+  },
+  "memory":  {
+    "total":  "2163961778176"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c"
+    },
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262"
+    },
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790"
+    },
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0"
+    }
+  ],
+  "cudaVersion":  "12.9",
+  "writerId":  "wykcz6se3w95mxueg1dbfpdz1rkcn7vb"
+}

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined/wandb/run-20260403_000132-nkf4iovm/logs/debug-core.log

@@ -0,0 +1,8 @@
+{"time":"2026-04-03T00:01:32.229543031+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpprztt27o/port-1047116.txt","pid":1047116,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-03T00:01:32.230033289+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1047116}
+{"time":"2026-04-03T00:01:32.230022361+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1047116-1057160-4156700530/socket","Net":"unix"}}
+{"time":"2026-04-03T00:01:32.416224439+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-03T00:01:32.430551403+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"nkf4iovm","id":"1(@)"}
+{"time":"2026-04-03T00:01:32.80024046+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"nkf4iovm","id":"1(@)"}
+{"time":"2026-04-03T00:01:38.985900014+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"rtqkkeh0iczo"}
+{"time":"2026-04-03T00:24:18.430668942+09:00","level":"INFO","msg":"server: parent process exited, terminating service process"}

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined/wandb/run-20260403_000132-nkf4iovm/logs/debug-internal.log

@@ -0,0 +1,6 @@
+{"time":"2026-04-03T00:01:32.430639522+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
+{"time":"2026-04-03T00:01:32.800089951+09:00","level":"INFO","msg":"stream: created new stream","id":"nkf4iovm"}
+{"time":"2026-04-03T00:01:32.800139938+09:00","level":"INFO","msg":"handler: started","stream_id":"nkf4iovm"}
+{"time":"2026-04-03T00:01:32.800233729+09:00","level":"INFO","msg":"stream: started","id":"nkf4iovm"}
+{"time":"2026-04-03T00:01:32.80025365+09:00","level":"INFO","msg":"sender: started","stream_id":"nkf4iovm"}
+{"time":"2026-04-03T00:01:32.800252986+09:00","level":"INFO","msg":"writer: started","stream_id":"nkf4iovm"}

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined/wandb/run-20260403_000132-nkf4iovm/logs/debug.log

@@ -0,0 +1,19 @@
+2026-04-03 00:01:32,169 INFO    MainThread:1047116 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
+2026-04-03 00:01:32,169 INFO    MainThread:1047116 [wandb_setup.py:_flush():81] Configure stats pid to 1047116
+2026-04-03 00:01:32,169 INFO    MainThread:1047116 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-03 00:01:32,169 INFO    MainThread:1047116 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined/wandb/run-20260403_000132-nkf4iovm/logs/debug.log
+2026-04-03 00:01:32,170 INFO    MainThread:1047116 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined/wandb/run-20260403_000132-nkf4iovm/logs/debug-internal.log
+2026-04-03 00:01:32,170 INFO    MainThread:1047116 [wandb_init.py:init():844] calling init triggers
+2026-04-03 00:01:32,170 INFO    MainThread:1047116 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf', 'name': 'sf_4gpu_bs8_lr2e6_5000iter_shift5_combined', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_lr1e5_10000iter_shift_5/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': "<class 'fastgen.callbacks.grad_clip.GradClipCallback'>"}, 'gpu_stats': {'every_n': '100', '_target_': "<class 'fastgen.callbacks.gpu_stats.GPUStatsCallback'>"}, 'train_profiler': {'every_n': '100', '_target_': "<class 'fastgen.callbacks.train_profiler.TrainProfilerCallback'>"}, 'param_count': {'_target_': "<class 'fastgen.callbacks.param_count.ParamCountCallback'>"}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': "<class 'fastgen.callbacks.ema.EMACallback'>"}, 'wandb': {'sample_logging_iter': '100', '_target_': "<class 'fastgen.callbacks.wandb.WandbCallback'>", 'fps': '25'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '5000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "<class 'fastgen.datasets.omniavatar_dataloader.OmniAvatarDataLoader'>"}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': '<function create_omniavatar_dataloader at 0x7f328019bce0>'}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "<class 'fastgen.networks.OmniAvatar.network_causal.CausalOmniAvatarWan'>"}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_phase2/step-10500.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "<class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>"}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt', 'merge_lora': 'False', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "<class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>"}, 'guidance_scale': '4.5', 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7f32803f2660>'}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.937', '0.833', '0.624', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '4', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'bfloat16', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7f32803f2660>'}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': "<class 'fastgen.networks.discriminators.Discriminator_EDM'>"}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7f32803f2660>'}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None}, 'model_class': {'config': None, '_target_': "<class 'fastgen.methods.omniavatar_self_forcing.OmniAvatarSelfForcingModel'>"}, '_wandb': {}}
+2026-04-03 00:01:32,170 INFO    MainThread:1047116 [wandb_init.py:init():892] starting backend
+2026-04-03 00:01:32,416 INFO    MainThread:1047116 [wandb_init.py:init():895] sending inform_init request
+2026-04-03 00:01:32,428 INFO    MainThread:1047116 [wandb_init.py:init():903] backend started and connected
+2026-04-03 00:01:32,432 INFO    MainThread:1047116 [wandb_init.py:init():973] updated telemetry
+2026-04-03 00:01:32,448 INFO    MainThread:1047116 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-04-03 00:01:33,668 INFO    MainThread:1047116 [wandb_init.py:init():1042] starting run threads in backend
+2026-04-03 00:01:33,982 INFO    MainThread:1047116 [wandb_run.py:_console_start():2524] atexit reg
+2026-04-03 00:01:33,982 INFO    MainThread:1047116 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-04-03 00:01:33,982 INFO    MainThread:1047116 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-04-03 00:01:33,982 INFO    MainThread:1047116 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-04-03 00:01:33,986 INFO    MainThread:1047116 [wandb_init.py:init():1082] run started, returning control to user process

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined/wandb/run-20260403_000132-nkf4iovm/run-nkf4iovm.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67b0ff79a5dafcc07f84b00216e9f32cb24f03c12ace3669ef628f85ac889c1e
+size 360448

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined/wandb_id.txt

@@ -0,0 +1 @@
+nkf4iovm

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined_v2/config.yaml

@@ -0,0 +1,259 @@
+dataloader_train:
+  _target_: <class 'fastgen.datasets.omniavatar_dataloader.OmniAvatarDataLoader'>
+  batch_size: '8'
+  data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt
+  latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png
+  neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt
+  num_workers: '2'
+  use_ref_sequence: 'True'
+dataloader_val:
+  _target_: <function create_omniavatar_dataloader at 0x7fb9c8423a60>
+  batch_size: '1'
+  data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt
+  latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png
+  load_ode_path: 'False'
+  neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt
+  num_workers: '2'
+  use_ref_sequence: 'True'
+eval:
+  max_ckpt: '100000000'
+  min_ckpt: '0'
+  num_samples: '50000'
+  samples_dir: samples
+  save_images: 'False'
+log_config:
+  group: omniavatar_sf
+  name: sf_4gpu_bs8_lr2e6_5000iter_shift5_combined_v2
+  project: OmniAvatar-FastGen
+  wandb_credential: ./credentials/wandb_api.txt
+  wandb_entity: paulhcho
+  wandb_mode: online
+model:
+  add_teacher_to_fsdp_dict: 'True'
+  context_noise: '0.0'
+  ddp_find_unused_parameters: 'True'
+  device: cuda
+  discriminator:
+    _target_: <class 'fastgen.networks.discriminators.Discriminator_EDM'>
+    all_res:
+    - '32'
+    - '16'
+    - '8'
+    feature_indices: '{0, 1, 2}'
+    in_channels: '256'
+  discriminator_optimizer:
+    _target_: <function get_optimizer at 0x7fb9c867e660>
+    betas:
+    - '0.9'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: '0.0001'
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  discriminator_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  enable_gradient_in_rollout: 'True'
+  enable_preprocessors: 'True'
+  fake_score: null
+  fake_score_net:
+    _target_: <class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors
+    in_dim: '65'
+    merge_lora: 'False'
+    mode: v2v
+    model_size: 1.3B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt
+    schedule_type: rf
+    use_audio: 'True'
+  fake_score_optimizer:
+    _target_: <function get_optimizer at 0x7fb9c867e660>
+    betas:
+    - '0.0'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: 2e-06
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  fake_score_pred_type: x0
+  fake_score_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  fsdp_meta_init: 'False'
+  gan_loss_weight_gen: '0'
+  gan_r1_reg_alpha: '0.1'
+  gan_r1_reg_weight: '0.0'
+  gan_use_same_t_noise: 'False'
+  grad_scaler_enabled: 'False'
+  grad_scaler_growth_interval: '2000'
+  grad_scaler_init_scale: '65536.0'
+  guidance_scale: '4.5'
+  input_shape:
+  - '16'
+  - '21'
+  - '64'
+  - '64'
+  last_step_only: 'False'
+  load_student_weights: 'False'
+  net:
+    _target_: <class 'fastgen.networks.OmniAvatar.network_causal.CausalOmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors
+    chunk_size: '3'
+    in_dim: '65'
+    mode: v2v
+    model_size: 1.3B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt
+    schedule_type: rf
+    total_num_frames: '21'
+    use_audio: 'True'
+  net_optimizer:
+    _target_: <function get_optimizer at 0x7fb9c867e660>
+    betas:
+    - '0.0'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: 2e-06
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  net_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  precision: bfloat16
+  precision_amp: null
+  precision_amp_enc: null
+  precision_amp_infer: null
+  precision_fsdp: bfloat16
+  pretrained_model_path: ''
+  pretrained_student_net_path: ''
+  same_step_across_blocks: 'True'
+  sample_t_cfg:
+    log_t_df: '0.01'
+    max_t: '0.999'
+    min_t: '0.001'
+    shift: '5.0'
+    t_list:
+    - '0.999'
+    - '0.937'
+    - '0.833'
+    - '0.624'
+    - '0.0'
+    time_dist_type: shifted
+    train_p_mean: '-1.1'
+    train_p_std: '2.0'
+  skip_layers: null
+  start_gradient_frame: '0'
+  student_sample_steps: '4'
+  student_sample_type: sde
+  student_update_freq: '5'
+  teacher:
+    _target_: <class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors
+    in_dim: '65'
+    merge_lora: 'True'
+    mode: v2v
+    model_size: 14B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_phase2/step-10500.pt
+    schedule_type: rf
+    use_audio: 'True'
+  use_ema: 'False'
+model_class:
+  _target_: <class 'fastgen.methods.omniavatar_self_forcing.OmniAvatarSelfForcingModel'>
+  config: null
+trainer:
+  augment_pipe: null
+  batch_size_global: null
+  callbacks:
+    ema:
+      _target_: <class 'fastgen.callbacks.ema.EMACallback'>
+      beta: '0.9999'
+      ema_halflife_kimg: '500'
+      ema_rampup_ratio: '0.05'
+      gamma: '16.97'
+      start_iter: '0'
+      type: constant
+    gpu_stats:
+      _target_: <class 'fastgen.callbacks.gpu_stats.GPUStatsCallback'>
+      every_n: '100'
+    grad_clip:
+      _target_: <class 'fastgen.callbacks.grad_clip.GradClipCallback'>
+      grad_norm: '10.0'
+      model_key: net
+    param_count:
+      _target_: <class 'fastgen.callbacks.param_count.ParamCountCallback'>
+    train_profiler:
+      _target_: <class 'fastgen.callbacks.train_profiler.TrainProfilerCallback'>
+      every_n: '100'
+    wandb:
+      _target_: <class 'fastgen.callbacks.wandb.WandbCallback'>
+      fps: '25'
+      sample_logging_iter: '100'
+  checkpointer:
+    pretrained_ckpt_key_map:
+      net: net
+    pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_lr1e5_10000iter_shift_5/checkpoints/0005000.pth
+    s3_container: s3://checkpoints/fastgen
+    s3_credential: ./credentials/s3.json
+    save_dir: /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_4gpu_bs8_lr2e6_5000iter_shift5_combined_v2/checkpoints
+    use_s3: 'False'
+  cudnn:
+    benchmark: 'True'
+    deterministic: 'False'
+  ddp: 'False'
+  fsdp: 'True'
+  fsdp_cpu_offload: 'False'
+  fsdp_min_num_params: '10000000'
+  fsdp_sharding_group_size: null
+  global_vars: null
+  global_vars_val:
+  - null
+  grad_accum_rounds: '2'
+  logging_iter: '1'
+  max_iter: '5000'
+  offload_module_in_decoding: 'False'
+  resume: 'False'
+  save_ckpt_iter: '100'
+  seed: '0'
+  skip_initial_validation: 'True'
+  tf32_enabled: 'True'
+  val_seed: null
+  validation_iter: '100'
+  visualize_teacher: 'False'

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_debug/config.yaml

@@ -0,0 +1,259 @@
+dataloader_train:
+  _target_: <class 'fastgen.datasets.omniavatar_dataloader.OmniAvatarDataLoader'>
+  batch_size: '8'
+  data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt
+  latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png
+  neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt
+  num_workers: '2'
+  use_ref_sequence: 'True'
+dataloader_val:
+  _target_: <function create_omniavatar_dataloader at 0x7f7c52b8fce0>
+  batch_size: '1'
+  data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt
+  latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png
+  load_ode_path: 'False'
+  neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt
+  num_workers: '2'
+  use_ref_sequence: 'True'
+eval:
+  max_ckpt: '100000000'
+  min_ckpt: '0'
+  num_samples: '50000'
+  samples_dir: samples
+  save_images: 'False'
+log_config:
+  group: omniavatar_sf
+  name: sf_combined_debug
+  project: OmniAvatar-FastGen
+  wandb_credential: ./credentials/wandb_api.txt
+  wandb_entity: paulhcho
+  wandb_mode: disabled
+model:
+  add_teacher_to_fsdp_dict: 'True'
+  context_noise: '0.0'
+  ddp_find_unused_parameters: 'True'
+  device: cuda
+  discriminator:
+    _target_: <class 'fastgen.networks.discriminators.Discriminator_EDM'>
+    all_res:
+    - '32'
+    - '16'
+    - '8'
+    feature_indices: '{0, 1, 2}'
+    in_channels: '256'
+  discriminator_optimizer:
+    _target_: <function get_optimizer at 0x7f7c52bf2660>
+    betas:
+    - '0.9'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: '0.0001'
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  discriminator_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  enable_gradient_in_rollout: 'True'
+  enable_preprocessors: 'True'
+  fake_score: null
+  fake_score_net:
+    _target_: <class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors
+    in_dim: '65'
+    merge_lora: 'False'
+    mode: v2v
+    model_size: 1.3B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt
+    schedule_type: rf
+    use_audio: 'True'
+  fake_score_optimizer:
+    _target_: <function get_optimizer at 0x7f7c52bf2660>
+    betas:
+    - '0.0'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: 2e-06
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  fake_score_pred_type: x0
+  fake_score_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  fsdp_meta_init: 'False'
+  gan_loss_weight_gen: '0'
+  gan_r1_reg_alpha: '0.1'
+  gan_r1_reg_weight: '0.0'
+  gan_use_same_t_noise: 'False'
+  grad_scaler_enabled: 'False'
+  grad_scaler_growth_interval: '2000'
+  grad_scaler_init_scale: '65536.0'
+  guidance_scale: '4.5'
+  input_shape:
+  - '16'
+  - '21'
+  - '64'
+  - '64'
+  last_step_only: 'False'
+  load_student_weights: 'False'
+  net:
+    _target_: <class 'fastgen.networks.OmniAvatar.network_causal.CausalOmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors
+    chunk_size: '3'
+    in_dim: '65'
+    mode: v2v
+    model_size: 1.3B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt
+    schedule_type: rf
+    total_num_frames: '21'
+    use_audio: 'True'
+  net_optimizer:
+    _target_: <function get_optimizer at 0x7f7c52bf2660>
+    betas:
+    - '0.0'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: 2e-06
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  net_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  precision: bfloat16
+  precision_amp: null
+  precision_amp_enc: null
+  precision_amp_infer: null
+  precision_fsdp: bfloat16
+  pretrained_model_path: ''
+  pretrained_student_net_path: ''
+  same_step_across_blocks: 'True'
+  sample_t_cfg:
+    log_t_df: '0.01'
+    max_t: '0.999'
+    min_t: '0.001'
+    shift: '5.0'
+    t_list:
+    - '0.999'
+    - '0.937'
+    - '0.833'
+    - '0.624'
+    - '0.0'
+    time_dist_type: shifted
+    train_p_mean: '-1.1'
+    train_p_std: '2.0'
+  skip_layers: null
+  start_gradient_frame: '0'
+  student_sample_steps: '4'
+  student_sample_type: sde
+  student_update_freq: '5'
+  teacher:
+    _target_: <class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors
+    in_dim: '65'
+    merge_lora: 'True'
+    mode: v2v
+    model_size: 14B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_phase2/step-10500.pt
+    schedule_type: rf
+    use_audio: 'True'
+  use_ema: 'False'
+model_class:
+  _target_: <class 'fastgen.methods.omniavatar_self_forcing.OmniAvatarSelfForcingModel'>
+  config: null
+trainer:
+  augment_pipe: null
+  batch_size_global: null
+  callbacks:
+    ema:
+      _target_: <class 'fastgen.callbacks.ema.EMACallback'>
+      beta: '0.9999'
+      ema_halflife_kimg: '500'
+      ema_rampup_ratio: '0.05'
+      gamma: '16.97'
+      start_iter: '0'
+      type: constant
+    gpu_stats:
+      _target_: <class 'fastgen.callbacks.gpu_stats.GPUStatsCallback'>
+      every_n: '100'
+    grad_clip:
+      _target_: <class 'fastgen.callbacks.grad_clip.GradClipCallback'>
+      grad_norm: '10.0'
+      model_key: net
+    param_count:
+      _target_: <class 'fastgen.callbacks.param_count.ParamCountCallback'>
+    train_profiler:
+      _target_: <class 'fastgen.callbacks.train_profiler.TrainProfilerCallback'>
+      every_n: '100'
+    wandb:
+      _target_: <class 'fastgen.callbacks.wandb.WandbCallback'>
+      fps: '25'
+      sample_logging_iter: '100'
+  checkpointer:
+    pretrained_ckpt_key_map:
+      net: net
+    pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_lr1e5_10000iter_shift_5/checkpoints/0005000.pth
+    s3_container: s3://checkpoints/fastgen
+    s3_credential: ./credentials/s3.json
+    save_dir: /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_debug/checkpoints
+    use_s3: 'False'
+  cudnn:
+    benchmark: 'True'
+    deterministic: 'False'
+  ddp: 'False'
+  fsdp: 'True'
+  fsdp_cpu_offload: 'False'
+  fsdp_min_num_params: '10000000'
+  fsdp_sharding_group_size: null
+  global_vars: null
+  global_vars_val:
+  - null
+  grad_accum_rounds: '2'
+  logging_iter: '1'
+  max_iter: '10'
+  offload_module_in_decoding: 'False'
+  resume: 'False'
+  save_ckpt_iter: '100'
+  seed: '0'
+  skip_initial_validation: 'True'
+  tf32_enabled: 'True'
+  val_seed: null
+  validation_iter: '100'
+  visualize_teacher: 'False'

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_debug/wandb_id.txt

@@ -0,0 +1 @@
+k4ws77lt

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/config.yaml

@@ -0,0 +1,259 @@
+dataloader_train:
+  _target_: <class 'fastgen.datasets.omniavatar_dataloader.OmniAvatarDataLoader'>
+  batch_size: '8'
+  data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt
+  latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png
+  neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt
+  num_workers: '2'
+  use_ref_sequence: 'True'
+dataloader_val:
+  _target_: <function create_omniavatar_dataloader at 0x7f99973a7ce0>
+  batch_size: '1'
+  data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt
+  latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png
+  load_ode_path: 'False'
+  neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt
+  num_workers: '2'
+  use_ref_sequence: 'True'
+eval:
+  max_ckpt: '100000000'
+  min_ckpt: '0'
+  num_samples: '50000'
+  samples_dir: samples
+  save_images: 'False'
+log_config:
+  group: omniavatar_sf
+  name: sf_combined_step_test
+  project: OmniAvatar-FastGen
+  wandb_credential: ./credentials/wandb_api.txt
+  wandb_entity: paulhcho
+  wandb_mode: online
+model:
+  add_teacher_to_fsdp_dict: 'True'
+  context_noise: '0.0'
+  ddp_find_unused_parameters: 'True'
+  device: cuda
+  discriminator:
+    _target_: <class 'fastgen.networks.discriminators.Discriminator_EDM'>
+    all_res:
+    - '32'
+    - '16'
+    - '8'
+    feature_indices: '{0, 1, 2}'
+    in_channels: '256'
+  discriminator_optimizer:
+    _target_: <function get_optimizer at 0x7f99975fe660>
+    betas:
+    - '0.9'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: '0.0001'
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  discriminator_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  enable_gradient_in_rollout: 'True'
+  enable_preprocessors: 'True'
+  fake_score: null
+  fake_score_net:
+    _target_: <class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors
+    in_dim: '65'
+    merge_lora: 'False'
+    mode: v2v
+    model_size: 1.3B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt
+    schedule_type: rf
+    use_audio: 'True'
+  fake_score_optimizer:
+    _target_: <function get_optimizer at 0x7f99975fe660>
+    betas:
+    - '0.0'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: 2e-06
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  fake_score_pred_type: x0
+  fake_score_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  fsdp_meta_init: 'False'
+  gan_loss_weight_gen: '0'
+  gan_r1_reg_alpha: '0.1'
+  gan_r1_reg_weight: '0.0'
+  gan_use_same_t_noise: 'False'
+  grad_scaler_enabled: 'False'
+  grad_scaler_growth_interval: '2000'
+  grad_scaler_init_scale: '65536.0'
+  guidance_scale: '4.5'
+  input_shape:
+  - '16'
+  - '21'
+  - '64'
+  - '64'
+  last_step_only: 'False'
+  load_student_weights: 'False'
+  net:
+    _target_: <class 'fastgen.networks.OmniAvatar.network_causal.CausalOmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors
+    chunk_size: '3'
+    in_dim: '65'
+    mode: v2v
+    model_size: 1.3B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt
+    schedule_type: rf
+    total_num_frames: '21'
+    use_audio: 'True'
+  net_optimizer:
+    _target_: <function get_optimizer at 0x7f99975fe660>
+    betas:
+    - '0.0'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: 2e-06
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  net_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  precision: bfloat16
+  precision_amp: null
+  precision_amp_enc: null
+  precision_amp_infer: null
+  precision_fsdp: bfloat16
+  pretrained_model_path: ''
+  pretrained_student_net_path: ''
+  same_step_across_blocks: 'True'
+  sample_t_cfg:
+    log_t_df: '0.01'
+    max_t: '0.999'
+    min_t: '0.001'
+    shift: '5.0'
+    t_list:
+    - '0.999'
+    - '0.937'
+    - '0.833'
+    - '0.624'
+    - '0.0'
+    time_dist_type: shifted
+    train_p_mean: '-1.1'
+    train_p_std: '2.0'
+  skip_layers: null
+  start_gradient_frame: '0'
+  student_sample_steps: '4'
+  student_sample_type: sde
+  student_update_freq: '5'
+  teacher:
+    _target_: <class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors
+    in_dim: '65'
+    merge_lora: 'True'
+    mode: v2v
+    model_size: 14B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_phase2/step-10500.pt
+    schedule_type: rf
+    use_audio: 'True'
+  use_ema: 'False'
+model_class:
+  _target_: <class 'fastgen.methods.omniavatar_self_forcing.OmniAvatarSelfForcingModel'>
+  config: null
+trainer:
+  augment_pipe: null
+  batch_size_global: null
+  callbacks:
+    ema:
+      _target_: <class 'fastgen.callbacks.ema.EMACallback'>
+      beta: '0.9999'
+      ema_halflife_kimg: '500'
+      ema_rampup_ratio: '0.05'
+      gamma: '16.97'
+      start_iter: '0'
+      type: constant
+    gpu_stats:
+      _target_: <class 'fastgen.callbacks.gpu_stats.GPUStatsCallback'>
+      every_n: '100'
+    grad_clip:
+      _target_: <class 'fastgen.callbacks.grad_clip.GradClipCallback'>
+      grad_norm: '10.0'
+      model_key: net
+    param_count:
+      _target_: <class 'fastgen.callbacks.param_count.ParamCountCallback'>
+    train_profiler:
+      _target_: <class 'fastgen.callbacks.train_profiler.TrainProfilerCallback'>
+      every_n: '100'
+    wandb:
+      _target_: <class 'fastgen.callbacks.wandb.WandbCallback'>
+      fps: '25'
+      sample_logging_iter: '100'
+  checkpointer:
+    pretrained_ckpt_key_map:
+      net: net
+    pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_lr1e5_10000iter_shift_5/checkpoints/0005000.pth
+    s3_container: s3://checkpoints/fastgen
+    s3_credential: ./credentials/s3.json
+    save_dir: /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/checkpoints
+    use_s3: 'False'
+  cudnn:
+    benchmark: 'True'
+    deterministic: 'False'
+  ddp: 'False'
+  fsdp: 'True'
+  fsdp_cpu_offload: 'False'
+  fsdp_min_num_params: '10000000'
+  fsdp_sharding_group_size: null
+  global_vars: null
+  global_vars_val:
+  - null
+  grad_accum_rounds: '2'
+  logging_iter: '1'
+  max_iter: '20'
+  offload_module_in_decoding: 'False'
+  resume: 'False'
+  save_ckpt_iter: '100'
+  seed: '0'
+  skip_initial_validation: 'True'
+  tf32_enabled: 'True'
+  val_seed: null
+  validation_iter: '100'
+  visualize_teacher: 'False'

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/wandb/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-04-02T22:45:31.396833745+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
+{"time":"2026-04-02T22:45:31.746696076+09:00","level":"INFO","msg":"stream: created new stream","id":"spcd04xe"}
+{"time":"2026-04-02T22:45:31.746744779+09:00","level":"INFO","msg":"handler: started","stream_id":"spcd04xe"}
+{"time":"2026-04-02T22:45:31.746822827+09:00","level":"INFO","msg":"stream: started","id":"spcd04xe"}
+{"time":"2026-04-02T22:45:31.746841154+09:00","level":"INFO","msg":"sender: started","stream_id":"spcd04xe"}
+{"time":"2026-04-02T22:45:31.74684523+09:00","level":"INFO","msg":"writer: started","stream_id":"spcd04xe"}
+{"time":"2026-04-02T22:52:57.91103952+09:00","level":"INFO","msg":"stream: closing","id":"spcd04xe"}
+{"time":"2026-04-02T22:52:58.486184439+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-04-02T22:52:58.869727008+09:00","level":"INFO","msg":"handler: closed","stream_id":"spcd04xe"}
+{"time":"2026-04-02T22:52:58.869816672+09:00","level":"INFO","msg":"sender: closed","stream_id":"spcd04xe"}
+{"time":"2026-04-02T22:52:58.869827326+09:00","level":"INFO","msg":"stream: closed","id":"spcd04xe"}

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/wandb/debug.log

@@ -0,0 +1,21 @@
+2026-04-02 22:45:31,146 INFO    MainThread:792541 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
+2026-04-02 22:45:31,146 INFO    MainThread:792541 [wandb_setup.py:_flush():81] Configure stats pid to 792541
+2026-04-02 22:45:31,146 INFO    MainThread:792541 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-02 22:45:31,146 INFO    MainThread:792541 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/wandb/run-20260402_224531-spcd04xe/logs/debug.log
+2026-04-02 22:45:31,146 INFO    MainThread:792541 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/wandb/run-20260402_224531-spcd04xe/logs/debug-internal.log
+2026-04-02 22:45:31,146 INFO    MainThread:792541 [wandb_init.py:init():844] calling init triggers
+2026-04-02 22:45:31,146 INFO    MainThread:792541 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf', 'name': 'sf_combined_step_test', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_lr1e5_10000iter_shift_5/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': "<class 'fastgen.callbacks.grad_clip.GradClipCallback'>"}, 'gpu_stats': {'every_n': '100', '_target_': "<class 'fastgen.callbacks.gpu_stats.GPUStatsCallback'>"}, 'train_profiler': {'every_n': '100', '_target_': "<class 'fastgen.callbacks.train_profiler.TrainProfilerCallback'>"}, 'param_count': {'_target_': "<class 'fastgen.callbacks.param_count.ParamCountCallback'>"}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': "<class 'fastgen.callbacks.ema.EMACallback'>"}, 'wandb': {'sample_logging_iter': '100', '_target_': "<class 'fastgen.callbacks.wandb.WandbCallback'>", 'fps': '25'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '20', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "<class 'fastgen.datasets.omniavatar_dataloader.OmniAvatarDataLoader'>"}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': '<function create_omniavatar_dataloader at 0x7ff2f56a3ce0>'}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "<class 'fastgen.networks.OmniAvatar.network_causal.CausalOmniAvatarWan'>"}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_phase2/step-10500.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "<class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>"}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt', 'merge_lora': 'False', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "<class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>"}, 'guidance_scale': '4.5', 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7ff2f5916660>'}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.937', '0.833', '0.624', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '4', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'bfloat16', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7ff2f5916660>'}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': "<class 'fastgen.networks.discriminators.Discriminator_EDM'>"}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7ff2f5916660>'}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None}, 'model_class': {'config': None, '_target_': "<class 'fastgen.methods.omniavatar_self_forcing.OmniAvatarSelfForcingModel'>"}, '_wandb': {}}
+2026-04-02 22:45:31,146 INFO    MainThread:792541 [wandb_init.py:init():892] starting backend
+2026-04-02 22:45:31,381 INFO    MainThread:792541 [wandb_init.py:init():895] sending inform_init request
+2026-04-02 22:45:31,394 INFO    MainThread:792541 [wandb_init.py:init():903] backend started and connected
+2026-04-02 22:45:31,398 INFO    MainThread:792541 [wandb_init.py:init():973] updated telemetry
+2026-04-02 22:45:31,413 INFO    MainThread:792541 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-04-02 22:45:32,719 INFO    MainThread:792541 [wandb_init.py:init():1042] starting run threads in backend
+2026-04-02 22:45:32,993 INFO    MainThread:792541 [wandb_run.py:_console_start():2524] atexit reg
+2026-04-02 22:45:32,993 INFO    MainThread:792541 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-04-02 22:45:32,994 INFO    MainThread:792541 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-04-02 22:45:32,994 INFO    MainThread:792541 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-04-02 22:45:32,998 INFO    MainThread:792541 [wandb_init.py:init():1082] run started, returning control to user process
+2026-04-02 22:52:57,911 INFO    wandb-AsyncioManager-main:792541 [service_client.py:_forward_responses():134] Reached EOF.
+2026-04-02 22:52:57,911 INFO    wandb-AsyncioManager-main:792541 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/wandb/run-20260402_224531-spcd04xe/files/config.yaml

@@ -0,0 +1,362 @@
+_wandb:
+    value:
+        cli_version: 0.25.0
+        e:
+            mf7qlulwtbceq6fkw52thj4sgf25e3dz:
+                args:
+                    - --config=fastgen/configs/experiments/OmniAvatar/config_sf.py
+                    - '-'
+                    - trainer.resume=False
+                    - log_config.name=sf_combined_step_test
+                    - log_config.project=OmniAvatar-FastGen
+                    - trainer.max_iter=20
+                codePath: train.py
+                codePathLocal: train.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.9"
+                disk:
+                    /:
+                        total: "1356758433792"
+                        used: "257963536384"
+                email: paul.hyunbin@gmail.com
+                executable: /home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12
+                git:
+                    commit: 04de80beaf50f849c12a55a5d8358d94530b7bb5
+                    remote: https://paulcho98:@github.com/paulcho98/FastGen.git
+                gpu: NVIDIA H200
+                gpu_count: 4
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "150754820096"
+                      name: NVIDIA H200
+                      uuid: GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "150754820096"
+                      name: NVIDIA H200
+                      uuid: GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "150754820096"
+                      name: NVIDIA H200
+                      uuid: GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "150754820096"
+                      name: NVIDIA H200
+                      uuid: GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0
+                host: main1
+                memory:
+                    total: "2163961778176"
+                os: Linux-5.15.0-151-generic-x86_64-with-glibc2.39
+                program: /home/work/.local/hyunbin/FastGen/train.py
+                python: CPython 3.12.12
+                root: /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test
+                startedAt: "2026-04-02T13:45:31.145420Z"
+                writerId: mf7qlulwtbceq6fkw52thj4sgf25e3dz
+        m: []
+        python_version: 3.12.12
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 50
+                - 53
+                - 63
+                - 71
+                - 83
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 50
+                - 53
+                - 63
+                - 71
+                - 83
+                - 98
+            "3":
+                - 13
+                - 14
+                - 16
+                - 61
+                - 62
+            "4": 3.12.12
+            "5": 0.25.0
+            "6": 5.3.0
+            "12": 0.25.0
+            "13": linux-x86_64
+dataloader_train:
+    value:
+        _target_: <class 'fastgen.datasets.omniavatar_dataloader.OmniAvatarDataLoader'>
+        batch_size: "8"
+        data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt
+        latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png
+        neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt
+        num_workers: "2"
+        use_ref_sequence: "True"
+dataloader_val:
+    value:
+        _target_: <function create_omniavatar_dataloader at 0x7ff2f56a3ce0>
+        batch_size: "1"
+        data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt
+        latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png
+        load_ode_path: "False"
+        neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt
+        num_workers: "2"
+        use_ref_sequence: "True"
+eval:
+    value:
+        max_ckpt: "100000000"
+        min_ckpt: "0"
+        num_samples: "50000"
+        samples_dir: samples
+        save_images: "False"
+log_config:
+    value:
+        group: omniavatar_sf
+        name: sf_combined_step_test
+        project: OmniAvatar-FastGen
+        wandb_credential: ./credentials/wandb_api.txt
+        wandb_entity: paulhcho
+        wandb_mode: online
+model:
+    value:
+        add_teacher_to_fsdp_dict: "True"
+        context_noise: "0.0"
+        ddp_find_unused_parameters: "True"
+        device: cuda
+        discriminator:
+            _target_: <class 'fastgen.networks.discriminators.Discriminator_EDM'>
+            all_res:
+                - "32"
+                - "16"
+                - "8"
+            feature_indices: '{0, 1, 2}'
+            in_channels: "256"
+        discriminator_optimizer:
+            _target_: <function get_optimizer at 0x7ff2f5916660>
+            betas:
+                - "0.9"
+                - "0.999"
+            eps: "1e-08"
+            fused: "False"
+            lr: "0.0001"
+            model: null
+            optim_type: adamw
+            weight_decay: "0.01"
+        discriminator_scheduler:
+            _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+            cycle_lengths:
+                - "10000000000"
+            f_max:
+                - "1.0"
+            f_min:
+                - "1.0"
+            f_start:
+                - "1e-06"
+            warm_up_steps:
+                - "0"
+        enable_gradient_in_rollout: "True"
+        enable_preprocessors: "True"
+        fake_score: null
+        fake_score_net:
+            _target_: <class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>
+            audio_hidden_size: "32"
+            base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors
+            in_dim: "65"
+            merge_lora: "False"
+            mode: v2v
+            model_size: 1.3B
+            net_pred_type: flow
+            omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt
+            schedule_type: rf
+            use_audio: "True"
+        fake_score_optimizer:
+            _target_: <function get_optimizer at 0x7ff2f5916660>
+            betas:
+                - "0.0"
+                - "0.999"
+            eps: "1e-08"
+            fused: "False"
+            lr: "2e-06"
+            model: null
+            optim_type: adamw
+            weight_decay: "0.01"
+        fake_score_pred_type: x0
+        fake_score_scheduler:
+            _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+            cycle_lengths:
+                - "10000000000"
+            f_max:
+                - "1.0"
+            f_min:
+                - "1.0"
+            f_start:
+                - "1e-06"
+            warm_up_steps:
+                - "0"
+        fsdp_meta_init: "False"
+        gan_loss_weight_gen: "0"
+        gan_r1_reg_alpha: "0.1"
+        gan_r1_reg_weight: "0.0"
+        gan_use_same_t_noise: "False"
+        grad_scaler_enabled: "False"
+        grad_scaler_growth_interval: "2000"
+        grad_scaler_init_scale: "65536.0"
+        guidance_scale: "4.5"
+        input_shape:
+            - "16"
+            - "21"
+            - "64"
+            - "64"
+        last_step_only: "False"
+        load_student_weights: "False"
+        net:
+            _target_: <class 'fastgen.networks.OmniAvatar.network_causal.CausalOmniAvatarWan'>
+            audio_hidden_size: "32"
+            base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors
+            chunk_size: "3"
+            in_dim: "65"
+            mode: v2v
+            model_size: 1.3B
+            net_pred_type: flow
+            omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt
+            schedule_type: rf
+            total_num_frames: "21"
+            use_audio: "True"
+        net_optimizer:
+            _target_: <function get_optimizer at 0x7ff2f5916660>
+            betas:
+                - "0.0"
+                - "0.999"
+            eps: "1e-08"
+            fused: "False"
+            lr: "2e-06"
+            model: null
+            optim_type: adamw
+            weight_decay: "0.01"
+        net_scheduler:
+            _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+            cycle_lengths:
+                - "10000000000"
+            f_max:
+                - "1.0"
+            f_min:
+                - "1.0"
+            f_start:
+                - "1e-06"
+            warm_up_steps:
+                - "0"
+        precision: bfloat16
+        precision_amp: null
+        precision_amp_enc: null
+        precision_amp_infer: null
+        precision_fsdp: bfloat16
+        pretrained_model_path: ""
+        pretrained_student_net_path: ""
+        same_step_across_blocks: "True"
+        sample_t_cfg:
+            log_t_df: "0.01"
+            max_t: "0.999"
+            min_t: "0.001"
+            shift: "5.0"
+            t_list:
+                - "0.999"
+                - "0.937"
+                - "0.833"
+                - "0.624"
+                - "0.0"
+            time_dist_type: shifted
+            train_p_mean: "-1.1"
+            train_p_std: "2.0"
+        skip_layers: null
+        start_gradient_frame: "0"
+        student_sample_steps: "4"
+        student_sample_type: sde
+        student_update_freq: "5"
+        teacher:
+            _target_: <class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>
+            audio_hidden_size: "32"
+            base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors
+            in_dim: "65"
+            merge_lora: "True"
+            mode: v2v
+            model_size: 14B
+            net_pred_type: flow
+            omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_phase2/step-10500.pt
+            schedule_type: rf
+            use_audio: "True"
+        use_ema: "False"
+model_class:
+    value:
+        _target_: <class 'fastgen.methods.omniavatar_self_forcing.OmniAvatarSelfForcingModel'>
+        config: null
+trainer:
+    value:
+        augment_pipe: null
+        batch_size_global: null
+        callbacks:
+            ema:
+                _target_: <class 'fastgen.callbacks.ema.EMACallback'>
+                beta: "0.9999"
+                ema_halflife_kimg: "500"
+                ema_rampup_ratio: "0.05"
+                gamma: "16.97"
+                start_iter: "0"
+                type: constant
+            gpu_stats:
+                _target_: <class 'fastgen.callbacks.gpu_stats.GPUStatsCallback'>
+                every_n: "100"
+            grad_clip:
+                _target_: <class 'fastgen.callbacks.grad_clip.GradClipCallback'>
+                grad_norm: "10.0"
+                model_key: net
+            param_count:
+                _target_: <class 'fastgen.callbacks.param_count.ParamCountCallback'>
+            train_profiler:
+                _target_: <class 'fastgen.callbacks.train_profiler.TrainProfilerCallback'>
+                every_n: "100"
+            wandb:
+                _target_: <class 'fastgen.callbacks.wandb.WandbCallback'>
+                fps: "25"
+                sample_logging_iter: "100"
+        checkpointer:
+            pretrained_ckpt_key_map:
+                net: net
+            pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_lr1e5_10000iter_shift_5/checkpoints/0005000.pth
+            s3_container: s3://checkpoints/fastgen
+            s3_credential: ./credentials/s3.json
+            save_dir: /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/checkpoints
+            use_s3: "False"
+        cudnn:
+            benchmark: "True"
+            deterministic: "False"
+        ddp: "False"
+        fsdp: "True"
+        fsdp_cpu_offload: "False"
+        fsdp_min_num_params: "10000000"
+        fsdp_sharding_group_size: null
+        global_vars: null
+        global_vars_val:
+            - null
+        grad_accum_rounds: "2"
+        logging_iter: "1"
+        max_iter: "20"
+        offload_module_in_decoding: "False"
+        resume: "False"
+        save_ckpt_iter: "100"
+        seed: "0"
+        skip_initial_validation: "True"
+        tf32_enabled: "True"
+        val_seed: null
+        validation_iter: "100"
+        visualize_teacher: "False"

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/wandb/run-20260402_224531-spcd04xe/files/output.log

@@ -0,0 +1,310 @@
+[Apr 2, 2026 - 22:45:32 | INFO | fastgen.callbacks.wandb:init_wandb:259 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable)
+[Apr 2, 2026 - 22:45:32 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully
+[Apr 2, 2026 - 22:45:33 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete
+[Apr 2, 2026 - 22:45:33 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer...
+[Apr 2, 2026 - 22:45:33 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully
+[Apr 2, 2026 - 22:45:33 | SUCCESS | __main__:main:33 ] Trainer initialized successfully
+[Apr 2, 2026 - 22:45:33 | INFO | fastgen.trainer:run:77 ] Starting training
+[Apr 2, 2026 - 22:45:33 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ...
+[Apr 2, 2026 - 22:45:33 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_lr1e5_10000iter_shift_5/checkpoints/0005000.pth
+[Apr 2, 2026 - 22:45:37 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict...
+[Apr 2, 2026 - 22:45:49 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: <All keys matched successfully>
+[Apr 2, 2026 - 22:45:49 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_lr1e5_10000iter_shift_5/checkpoints/0005000.pth at iteration 5000
+[Apr 2, 2026 - 22:45:49 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 5000.
+[Apr 2, 2026 - 22:45:49 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ...
+[Apr 2, 2026 - 22:45:49 | INFO | fastgen.methods.model:on_train_begin:296 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True
+[Apr 2, 2026 - 22:45:49 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed
+[Apr 2, 2026 - 22:45:49 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp ..
+[Apr 2, 2026 - 22:45:49 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks...
+[Apr 2, 2026 - 22:45:49 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)...
+[Apr 2, 2026 - 22:45:49 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:1950 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics)
+[Apr 2, 2026 - 22:45:49 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding
+[Apr 2, 2026 - 22:45:50 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 1.2s
+[Apr 2, 2026 - 22:45:50 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.60B params)...
+[Apr 2, 2026 - 22:45:50 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:765 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection)
+[Apr 2, 2026 - 22:45:51 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding
+[Apr 2, 2026 - 22:45:51 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 1.5s
+[Apr 2, 2026 - 22:45:51 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)...
+[Apr 2, 2026 - 22:45:51 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:765 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection)
+[Apr 2, 2026 - 22:46:02 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding
+[Apr 2, 2026 - 22:46:02 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 11.0s
+[Apr 2, 2026 - 22:46:02 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed
+[Apr 2, 2026 - 22:46:02 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback.
+[Apr 2, 2026 - 22:46:02 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None
+[Apr 2, 2026 - 22:46:02 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 0.
+[Apr 2, 2026 - 22:46:02 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader...
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:361 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingModel) has 1596.36 M trainable and 17311.83 M total params (logical).
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingModel) has 420.42 M trainable and 4350.43 M total params LOCAL on rank 0.
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingModel) has 420.14 M trainable and 4348.63 M total params LOCAL on rank 1.
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingModel) has 420.14 M trainable and 4348.63 M total params LOCAL on rank 2.
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingModel) has 420.14 M trainable and 4348.63 M total params LOCAL on rank 3.
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical).
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.68 M trainable and 376.68 M total params LOCAL on rank 0.
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 1.
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 2.
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 3.
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 174.98 M trainable and 1596.36 M total params (logical).
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.30 M total params LOCAL on rank 0.
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 1.
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 2.
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 3.
+[Apr 2, 2026 - 22:46:08 | INFO | fastgen.trainer:run:174 ] iter_start: 0
+[MEM] fake_score_update: START: alloc=9.45GB reserved=9.88GB peak=9.60GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=12.28GB reserved=49.39GB peak=45.74GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=13.16GB peak=45.74GB
+[MEM-fwd] after block 0: alloc=13.74GB peak=45.74GB
+[MEM-fwd] after block 10: alloc=19.55GB peak=45.74GB
+[MEM-fwd] after block 20: alloc=24.84GB peak=45.74GB
+[MEM-fwd] after block 29: alloc=29.59GB peak=45.74GB
+[MEM-fwd] after head+unpatchify: alloc=30.67GB peak=45.74GB
+[MEM] fake_score_update: START: alloc=13.25GB reserved=15.24GB peak=54.53GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=13.29GB reserved=47.38GB peak=46.75GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=14.17GB peak=46.75GB
+[MEM-fwd] after block 0: alloc=14.75GB peak=46.75GB
+[MEM-fwd] after block 10: alloc=20.56GB peak=46.75GB
+[MEM-fwd] after block 20: alloc=25.85GB peak=46.75GB
+[MEM-fwd] after block 29: alloc=30.61GB peak=46.75GB
+[MEM-fwd] after head+unpatchify: alloc=31.69GB peak=46.75GB
+[Apr 2, 2026 - 22:47:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg        Max        Min
+cpu_mem_gb                38.492574  38.558720  38.459610
+peak_gpu_mem_gb           51.700073  51.700073  51.700073
+peak_gpu_mem_reserved_gb  53.640625  53.640625  53.640625
+util                      90.500000  97.000000  86.000000
+[Apr 2, 2026 - 22:47:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 1 : data loading time        0.81
+[Apr 2, 2026 - 22:47:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 1 : avg forward pass time    15.15
+[Apr 2, 2026 - 22:47:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 1 : backward pass time       11.36
+[Apr 2, 2026 - 22:47:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 1 : optimizer step time      1.28
+[Apr 2, 2026 - 22:47:05 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 1--------------------
+[Apr 2, 2026 - 22:47:05 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.0068        iter count: 1.0
+[Apr 2, 2026 - 22:47:05 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0068   iter count: 1.0
+[Apr 2, 2026 - 22:47:05 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_disc: 0.0000     iter count: 1.0
+[Apr 2, 2026 - 22:47:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=13.43GB reserved=57.60GB peak=55.51GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=13.47GB reserved=47.47GB peak=46.93GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=14.35GB peak=46.93GB
+[MEM-fwd] after block 0: alloc=14.93GB peak=46.93GB
+[MEM-fwd] after block 10: alloc=20.74GB peak=46.93GB
+[MEM-fwd] after block 20: alloc=26.02GB peak=46.93GB
+[MEM-fwd] after block 29: alloc=30.78GB peak=46.93GB
+[MEM-fwd] after head+unpatchify: alloc=31.86GB peak=46.93GB
+[MEM] fake_score_update: START: alloc=14.21GB reserved=58.37GB peak=55.69GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=14.25GB reserved=48.25GB peak=47.71GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=15.13GB peak=47.71GB
+[MEM-fwd] after block 0: alloc=15.71GB peak=47.71GB
+[MEM-fwd] after block 10: alloc=21.52GB peak=47.71GB
+[MEM-fwd] after block 20: alloc=26.81GB peak=47.71GB
+[MEM-fwd] after block 29: alloc=31.57GB peak=47.71GB
+[MEM-fwd] after head+unpatchify: alloc=32.65GB peak=47.71GB
+[Apr 2, 2026 - 22:47:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg         Max        Min
+cpu_mem_gb                38.666036   38.732494  38.632969
+peak_gpu_mem_gb           52.593685   52.593685  52.593685
+peak_gpu_mem_reserved_gb  54.365234   54.365234  54.365234
+util                      96.750000  100.000000  91.000000
+[Apr 2, 2026 - 22:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 2 : avg iteration time       51.61 seconds
+[Apr 2, 2026 - 22:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 2 : data loading time        0.00
+[Apr 2, 2026 - 22:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 2 : avg forward pass time    13.83
+[Apr 2, 2026 - 22:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 2 : backward pass time       11.39
+[Apr 2, 2026 - 22:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 2 : optimizer step time      1.16
+[Apr 2, 2026 - 22:47:57 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 2--------------------
+[Apr 2, 2026 - 22:47:57 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.0070        iter count: 1.0
+[Apr 2, 2026 - 22:47:57 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0070   iter count: 1.0
+[Apr 2, 2026 - 22:47:57 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_disc: 0.0000     iter count: 1.0
+[Apr 2, 2026 - 22:47:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=13.43GB reserved=58.37GB peak=56.47GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=13.47GB reserved=47.49GB peak=46.93GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=14.35GB peak=46.93GB
+[MEM-fwd] after block 0: alloc=14.93GB peak=46.93GB
+[MEM-fwd] after block 10: alloc=20.74GB peak=46.93GB
+[MEM-fwd] after block 20: alloc=26.02GB peak=46.93GB
+[MEM-fwd] after block 29: alloc=30.78GB peak=46.93GB
+[MEM-fwd] after head+unpatchify: alloc=31.86GB peak=46.93GB
+[MEM] fake_score_update: START: alloc=14.21GB reserved=58.40GB peak=55.69GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=14.25GB reserved=48.29GB peak=47.72GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=15.13GB peak=47.72GB
+[MEM-fwd] after block 0: alloc=15.71GB peak=47.72GB
+[MEM-fwd] after block 10: alloc=21.52GB peak=47.72GB
+[MEM-fwd] after block 20: alloc=26.81GB peak=47.72GB
+[MEM-fwd] after block 29: alloc=31.57GB peak=47.72GB
+[MEM-fwd] after head+unpatchify: alloc=32.65GB peak=47.72GB
+[Apr 2, 2026 - 22:48:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg         Max        Min
+cpu_mem_gb                38.666379   38.733353  38.633148
+peak_gpu_mem_gb           52.593685   52.593685  52.593685
+peak_gpu_mem_reserved_gb  54.404297   54.404297  54.404297
+util                      98.000000  100.000000  94.000000
+[Apr 2, 2026 - 22:48:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 3 : avg iteration time       57.83 seconds
+[Apr 2, 2026 - 22:48:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 3 : data loading time        0.00
+[Apr 2, 2026 - 22:48:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 3 : avg forward pass time    16.98
+[Apr 2, 2026 - 22:48:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 3 : backward pass time       11.32
+[Apr 2, 2026 - 22:48:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 3 : optimizer step time      1.16
+[Apr 2, 2026 - 22:48:55 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 3--------------------
+[Apr 2, 2026 - 22:48:55 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.0072        iter count: 1.0
+[Apr 2, 2026 - 22:48:55 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0072   iter count: 1.0
+[Apr 2, 2026 - 22:48:55 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_disc: 0.0000     iter count: 1.0
+[Apr 2, 2026 - 22:48:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=13.43GB reserved=58.42GB peak=56.47GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=13.47GB reserved=47.52GB peak=46.93GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=14.35GB peak=46.93GB
+[MEM-fwd] after block 0: alloc=14.93GB peak=46.93GB
+[MEM-fwd] after block 10: alloc=20.74GB peak=46.93GB
+[MEM-fwd] after block 20: alloc=26.02GB peak=46.93GB
+[MEM-fwd] after block 29: alloc=30.78GB peak=46.93GB
+[MEM-fwd] after head+unpatchify: alloc=31.86GB peak=46.93GB
+[MEM] fake_score_update: START: alloc=14.21GB reserved=58.42GB peak=55.69GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=14.25GB reserved=48.27GB peak=47.72GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=15.13GB peak=47.72GB
+[MEM-fwd] after block 0: alloc=15.71GB peak=47.72GB
+[MEM-fwd] after block 10: alloc=21.52GB peak=47.72GB
+[MEM-fwd] after block 20: alloc=26.81GB peak=47.72GB
+[MEM-fwd] after block 29: alloc=31.57GB peak=47.72GB
+[MEM-fwd] after head+unpatchify: alloc=32.65GB peak=47.72GB
+[Apr 2, 2026 - 22:49:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats:
+                                Avg        Max        Min
+cpu_mem_gb                38.666442  38.733406  38.633190
+peak_gpu_mem_gb           52.593685  52.593685  52.593685
+peak_gpu_mem_reserved_gb  54.443359  54.443359  54.443359
+util                      96.500000  99.000000  92.000000
+[Apr 2, 2026 - 22:49:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 4 : avg iteration time       57.83 seconds
+[Apr 2, 2026 - 22:49:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 4 : data loading time        0.00
+[Apr 2, 2026 - 22:49:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 4 : avg forward pass time    16.97
+[Apr 2, 2026 - 22:49:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 4 : backward pass time       11.34
+[Apr 2, 2026 - 22:49:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 4 : optimizer step time      1.19
+[Apr 2, 2026 - 22:49:53 | INFO | fastgen.callbacks.wandb:log_stats:569 ] logging train stats at iteration 4--------------------
+[Apr 2, 2026 - 22:49:53 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_total_loss: 0.0073        iter count: 1.0
+[Apr 2, 2026 - 22:49:53 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_fake_score_loss: 0.0073   iter count: 1.0
+[Apr 2, 2026 - 22:49:53 | INFO | fastgen.callbacks.wandb:get_stat:303 ] avg_gan_loss_disc: 0.0000     iter count: 1.0
+[Apr 2, 2026 - 22:49:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:600 ] WandB logging complete after 0.00 seconds
+[MEM] fake_score_update: START: alloc=13.43GB reserved=58.46GB peak=56.47GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=13.47GB reserved=47.49GB peak=46.93GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=14.35GB peak=46.93GB
+[MEM-fwd] after block 0: alloc=14.93GB peak=46.93GB
+[MEM-fwd] after block 10: alloc=20.74GB peak=46.93GB
+[MEM-fwd] after block 20: alloc=26.02GB peak=46.93GB
+[MEM-fwd] after block 29: alloc=30.78GB peak=46.93GB
+[MEM-fwd] after head+unpatchify: alloc=31.86GB peak=46.93GB
+[MEM] student_update: START: alloc=14.32GB reserved=58.46GB peak=55.75GB
+[MEM] student_update: after rollout: alloc=63.85GB reserved=66.24GB peak=65.62GB
+[MEM] student_update: after perturb: alloc=63.87GB reserved=66.24GB peak=65.62GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=64.62GB peak=65.62GB
+[MEM-fwd] after block 0: alloc=64.67GB peak=75.08GB
+[MEM-fwd] after block 10: alloc=65.20GB peak=75.61GB
+[MEM-fwd] after block 20: alloc=65.20GB peak=75.61GB
+[MEM-fwd] after block 29: alloc=65.20GB peak=75.61GB
+[MEM-fwd] after head+unpatchify: alloc=64.69GB peak=75.61GB
+[MEM] student_update: after fake_score: alloc=63.95GB reserved=78.02GB peak=75.61GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 5120]), 40 blocks): alloc=66.83GB peak=75.61GB
+[MEM-fwd] after block 0: alloc=67.05GB peak=88.90GB
+[MEM-fwd] after block 10: alloc=68.82GB peak=90.66GB
+[MEM-fwd] after block 20: alloc=68.82GB peak=90.66GB
+[MEM-fwd] after block 30: alloc=68.82GB peak=90.66GB
+[MEM-fwd] after block 39: alloc=68.82GB peak=90.66GB
+[MEM-fwd] after head+unpatchify: alloc=67.08GB peak=90.66GB
+[MEM] student_update: after teacher: alloc=64.45GB reserved=93.79GB peak=90.66GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 5120]), 40 blocks): alloc=66.37GB peak=90.66GB
+[MEM-fwd] after block 0: alloc=67.08GB peak=90.66GB
+[MEM-fwd] after block 10: alloc=68.84GB peak=90.68GB
+[MEM-fwd] after block 20: alloc=68.84GB peak=90.68GB
+[MEM-fwd] after block 30: alloc=68.84GB peak=90.68GB
+[MEM-fwd] after block 39: alloc=68.84GB peak=90.68GB
+[MEM-fwd] after head+unpatchify: alloc=67.10GB peak=90.68GB
+[MEM] student_update: after CFG: alloc=64.45GB reserved=93.83GB peak=90.68GB
+[MEM] student_update: after VSD loss: alloc=64.47GB reserved=93.83GB peak=90.68GB
+[MEM] fake_score_update: START: alloc=12.68GB reserved=72.00GB peak=90.68GB
+[MEM] fake_score_update: after student gen (no_grad): alloc=15.51GB reserved=52.62GB peak=48.98GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=16.28GB peak=48.98GB
+[MEM-fwd] after block 0: alloc=16.91GB peak=48.98GB
+[MEM-fwd] after block 10: alloc=22.73GB peak=48.98GB
+[MEM-fwd] after block 20: alloc=28.01GB peak=48.98GB
+[MEM-fwd] after block 29: alloc=32.77GB peak=48.98GB
+[MEM-fwd] after head+unpatchify: alloc=33.85GB peak=48.98GB
+[MEM] student_update: START: alloc=15.52GB reserved=61.80GB peak=56.95GB
+[MEM] student_update: after rollout: alloc=65.05GB reserved=68.04GB peak=66.83GB
+[MEM] student_update: after perturb: alloc=65.07GB reserved=68.04GB peak=66.83GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 1536]), 30 blocks): alloc=65.82GB peak=66.83GB
+[MEM-fwd] after block 0: alloc=65.87GB peak=76.28GB
+[MEM-fwd] after block 10: alloc=66.40GB peak=76.81GB
+[MEM-fwd] after block 20: alloc=66.40GB peak=76.81GB
+[MEM-fwd] after block 29: alloc=66.40GB peak=76.81GB
+[MEM-fwd] after head+unpatchify: alloc=65.90GB peak=76.81GB
+[MEM] student_update: after fake_score: alloc=65.15GB reserved=79.81GB peak=76.81GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 5120]), 40 blocks): alloc=67.08GB peak=76.81GB
+[MEM-fwd] after block 0: alloc=67.78GB peak=89.62GB
+[MEM-fwd] after block 10: alloc=69.54GB peak=91.39GB
+[MEM-fwd] after block 20: alloc=69.54GB peak=91.39GB
+[MEM-fwd] after block 30: alloc=69.54GB peak=91.39GB
+[MEM-fwd] after block 39: alloc=69.54GB peak=91.39GB
+[MEM-fwd] after head+unpatchify: alloc=67.80GB peak=91.39GB
+[MEM] student_update: after teacher: alloc=65.17GB reserved=94.89GB peak=91.39GB
+[MEM-fwd] layer_loop_start (dim=torch.Size([8, 21504, 5120]), 40 blocks): alloc=67.10GB peak=91.39GB
+[MEM-fwd] after block 0: alloc=67.80GB peak=91.39GB
+[MEM-fwd] after block 10: alloc=69.57GB peak=91.41GB
+[MEM-fwd] after block 20: alloc=69.57GB peak=91.41GB
+[MEM-fwd] after block 30: alloc=69.57GB peak=91.41GB
+[MEM-fwd] after block 39: alloc=69.57GB peak=91.41GB
+[MEM-fwd] after head+unpatchify: alloc=67.83GB peak=91.41GB
+[MEM] student_update: after CFG: alloc=65.17GB reserved=94.91GB peak=91.41GB
+[MEM] student_update: after VSD loss: alloc=65.20GB reserved=94.91GB peak=91.41GB
+Traceback (most recent call last):
+  File "/home/work/.local/hyunbin/FastGen/train.py", line 46, in <module>
+    main(config)
+  File "/home/work/.local/hyunbin/FastGen/train.py", line 37, in main
+    fastgen_trainer.run(model)
+  File "/home/work/.local/hyunbin/FastGen/fastgen/trainer.py", line 194, in run
+    loss_map, outputs = self.train_step(model_ddp, model, data, iter_cur, grad_accum_iter)
+                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/work/.local/hyunbin/FastGen/fastgen/trainer.py", line 331, in train_step
+    model.grad_scaler.scale(loss_map["total_loss"] / grad_accum_rounds).backward()
+  File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/_tensor.py", line 630, in backward
+    torch.autograd.backward(
+  File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/autograd/__init__.py", line 364, in backward
+    _engine_run_backward(
+  File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/autograd/graph.py", line 865, in _engine_run_backward
+    return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/utils/checkpoint.py", line 1177, in unpack_hook
+    frame.check_recomputed_tensors_match(gid)
+  File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/utils/checkpoint.py", line 882, in check_recomputed_tensors_match
+    raise CheckpointError(
+torch.utils.checkpoint.CheckpointError: torch.utils.checkpoint: A different number of tensors was saved during the original forward and recomputation.
+Number of tensors saved during forward: 94
+Number of tensors saved during recomputation: 80.
+
+Tip: To see a more detailed error message, either pass `debug=True` to
+`torch.utils.checkpoint.checkpoint(...)` or wrap the code block
+with `with torch.utils.checkpoint.set_checkpoint_debug_enabled(True):` to
+enable checkpoint‑debug mode globally.
+
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/home/work/.local/hyunbin/FastGen/train.py", line 46, in <module>
+[rank0]:     main(config)
+[rank0]:   File "/home/work/.local/hyunbin/FastGen/train.py", line 37, in main
+[rank0]:     fastgen_trainer.run(model)
+[rank0]:   File "/home/work/.local/hyunbin/FastGen/fastgen/trainer.py", line 194, in run
+[rank0]:     loss_map, outputs = self.train_step(model_ddp, model, data, iter_cur, grad_accum_iter)
+[rank0]:                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/home/work/.local/hyunbin/FastGen/fastgen/trainer.py", line 331, in train_step
+[rank0]:     model.grad_scaler.scale(loss_map["total_loss"] / grad_accum_rounds).backward()
+[rank0]:   File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/_tensor.py", line 630, in backward
+[rank0]:     torch.autograd.backward(
+[rank0]:   File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/autograd/__init__.py", line 364, in backward
+[rank0]:     _engine_run_backward(
+[rank0]:   File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/autograd/graph.py", line 865, in _engine_run_backward
+[rank0]:     return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/utils/checkpoint.py", line 1177, in unpack_hook
+[rank0]:     frame.check_recomputed_tensors_match(gid)
+[rank0]:   File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/utils/checkpoint.py", line 882, in check_recomputed_tensors_match
+[rank0]:     raise CheckpointError(
+[rank0]: torch.utils.checkpoint.CheckpointError: torch.utils.checkpoint: A different number of tensors was saved during the original forward and recomputation.
+[rank0]: Number of tensors saved during forward: 94
+[rank0]: Number of tensors saved during recomputation: 80.
+
+[rank0]: Tip: To see a more detailed error message, either pass `debug=True` to
+[rank0]: `torch.utils.checkpoint.checkpoint(...)` or wrap the code block
+[rank0]: with `with torch.utils.checkpoint.set_checkpoint_debug_enabled(True):` to
+[rank0]: enable checkpoint‑debug mode globally.

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/wandb/run-20260402_224531-spcd04xe/files/requirements.txt

@@ -0,0 +1,222 @@
+fastgen==0.1.0
+nvitop==1.6.1
+ftfy==6.3.1
+braceexpand==0.1.7
+antlr4-python3-runtime==4.9.3
+webdataset==1.0.2
+sentry-sdk==2.53.0
+rdkit==2025.9.5
+python-dotenv==1.2.1
+proglog==0.1.12
+omegaconf==2.3.0
+narwhals==2.17.0
+loguru==0.7.3
+imageio-ffmpeg==0.6.0
+plotly==6.5.2
+moviepy==2.2.1
+hydra-core==1.3.2
+wandb==0.25.0
+fastgen==0.1.0
+packaging==25.0
+setuptools==80.10.2
+wheel==0.46.3
+pip==26.0.1
+webencodings==0.5.1
+pure_eval==0.2.3
+ptyprocess==0.7.0
+nvidia-ml-py==13.590.48
+nvidia-cusparselt-cu12==0.7.1
+mpmath==1.3.0
+fastjsonschema==2.21.2
+zipp==3.23.0
+xyzservices==2025.11.0
+widgetsnbextension==4.0.15
+websocket-client==1.9.0
+webcolors==25.10.0
+wcwidth==0.6.0
+urllib3==2.6.3
+uri-template==1.3.0
+tzdata==2025.3
+typing_extensions==4.15.0
+triton==3.6.0
+traitlets==5.14.3
+tqdm==4.67.3
+tornado==6.5.5
+tinycss2==1.4.0
+sympy==1.14.0
+soupsieve==2.8.3
+smmap==5.0.3
+six==1.16.0
+sentencepiece==0.2.1
+Send2Trash==2.1.0
+safetensors==0.7.0
+rpds-py==0.30.0
+rfc3986-validator==0.1.1
+regex==2026.2.28
+pyzmq==27.1.0
+PyYAML==6.0.3
+python-json-logger==4.0.0
+Pygments==2.19.2
+pycparser==3.0
+psutil==7.2.2
+protobuf==4.24.4
+prometheus_client==0.24.1
+platformdirs==4.9.4
+pillow==11.3.0
+pexpect==4.9.0
+parso==0.8.6
+pandocfilters==1.5.1
+nvidia-nvtx-cu12==12.8.90
+nvidia-nvshmem-cu12==3.4.5
+nvidia-nvjitlink-cu12==12.8.93
+nvidia-nccl-cu12==2.27.5
+nvidia-curand-cu12==10.3.9.90
+nvidia-cufile-cu12==1.13.1.3
+nvidia-cuda-runtime-cu12==12.8.90
+nvidia-cuda-nvrtc-cu12==12.8.93
+nvidia-cuda-cupti-cu12==12.8.90
+nvidia-cublas-cu12==12.8.4.1
+numpy==1.26.4
+networkx==3.6.1
+nest-asyncio==1.6.0
+mistune==3.2.0
+MarkupSafe==3.0.3
+lark==1.3.1
+jupyterlab_widgets==3.0.16
+jupyterlab_pygments==0.3.0
+jsonpointer==3.0.0
+json5==0.13.0
+jmespath==1.1.0
+idna==3.11
+hf-xet==1.4.2
+h11==0.16.0
+fsspec==2026.2.0
+fqdn==1.5.1
+filelock==3.25.2
+executing==2.2.1
+einops==0.8.2
+defusedxml==0.7.1
+decorator==5.2.1
+debugpy==1.8.20
+cuda-pathfinder==1.4.2
+comm==0.2.3
+click==8.3.1
+charset-normalizer==3.4.5
+certifi==2026.2.25
+bleach==6.3.0
+babel==2.18.0
+av==17.0.0
+attrs==25.4.0
+async-lru==2.2.0
+asttokens==3.0.1
+annotated-types==0.7.0
+typing-inspection==0.4.2
+terminado==0.18.1
+stack-data==0.6.3
+scipy==1.17.1
+rfc3987-syntax==1.1.0
+rfc3339-validator==0.1.4
+requests==2.32.5
+referencing==0.37.0
+python-dateutil==2.9.0.post0
+pydantic_core==2.41.5
+prompt_toolkit==3.0.52
+opencv-python-headless==4.11.0.86
+nvidia-cusparse-cu12==12.5.8.93
+nvidia-cufft-cu12==11.3.3.83
+nvidia-cudnn-cu12==9.10.2.21
+matplotlib-inline==0.2.1
+jupyter_core==5.9.1
+Jinja2==3.1.6
+jedi==0.19.2
+ipython_pygments_lexers==1.1.1
+importlib_metadata==8.7.1
+ImageIO==2.37.3
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==12.9.4
+contourpy==1.3.3
+cffi==2.0.0
+beautifulsoup4==4.14.3
+anyio==4.12.1
+soundfile==0.13.1
+pydantic==2.12.5
+nvidia-cusolver-cu12==11.7.3.90
+jupyter_server_terminals==0.5.4
+jupyter_client==8.8.0
+jsonschema-specifications==2025.9.1
+ipython==9.11.0
+httpx==0.28.1
+GitPython==3.1.46
+botocore==1.42.68
+bokeh==3.9.0
+arrow==1.4.0
+argon2-cffi-bindings==25.1.0
+torch==2.10.0
+s3transfer==0.16.0
+jsonschema==4.26.0
+isoduration==20.11.0
+ipywidgets==8.1.8
+ipykernel==7.2.0
+argon2-cffi==25.1.0
+torchvision==0.25.0
+nbformat==5.10.4
+jupyter-console==6.6.3
+boto3==1.42.68
+accelerate==1.13.0
+nbclient==0.10.4
+jupyter-events==0.12.0
+nbconvert==7.17.0
+jupyter_server==2.17.0
+notebook_shim==0.2.4
+jupyterlab_server==2.28.0
+jupyter-lsp==2.3.0
+jupyterlab==4.5.6
+notebook==7.5.5
+jupyter==1.1.1
+fastgen==0.1.0
+pandas==3.0.1
+shellingham==1.5.4
+mdurl==0.1.2
+annotated-doc==0.0.4
+markdown-it-py==4.0.0
+rich==14.3.3
+typer==0.24.1
+huggingface_hub==1.7.1
+timm==1.0.25
+tokenizers==0.22.2
+diffusers==0.37.0
+transformers==5.3.0
+peft==0.18.1
+easydict==1.13
+lmdb==2.2.0
+threadpoolctl==3.6.0
+soxr==1.0.0
+msgpack==1.1.2
+llvmlite==0.47.0
+lazy-loader==0.5
+joblib==1.5.3
+audioread==3.1.0
+scikit-learn==1.8.0
+pooch==1.9.0
+numba==0.65.0
+librosa==0.11.0
+simsimd==6.5.16
+flatbuffers==25.12.19
+tifffile==2026.3.3
+stringzilla==4.6.0
+pyparsing==3.3.2
+prettytable==3.17.0
+onnx==1.17.0
+kiwisolver==1.5.0
+fonttools==4.62.1
+Cython==3.2.4
+cycler==0.12.1
+scikit-image==0.26.0
+onnxruntime==1.24.4
+matplotlib==3.10.8
+albucore==0.0.24
+albumentations==2.0.8
+insightface==0.7.3
+kornia_rs==0.1.10
+kornia==0.8.2

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/wandb/run-20260402_224531-spcd04xe/files/wandb-metadata.json

@@ -0,0 +1,69 @@
+{
+  "os":  "Linux-5.15.0-151-generic-x86_64-with-glibc2.39",
+  "python":  "CPython 3.12.12",
+  "startedAt":  "2026-04-02T13:45:31.145420Z",
+  "args":  [
+    "--config=fastgen/configs/experiments/OmniAvatar/config_sf.py",
+    "-",
+    "trainer.resume=False",
+    "log_config.name=sf_combined_step_test",
+    "log_config.project=OmniAvatar-FastGen",
+    "trainer.max_iter=20"
+  ],
+  "program":  "/home/work/.local/hyunbin/FastGen/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://paulcho98:@github.com/paulcho98/FastGen.git",
+    "commit":  "04de80beaf50f849c12a55a5d8358d94530b7bb5"
+  },
+  "email":  "paul.hyunbin@gmail.com",
+  "root":  "/tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test",
+  "host":  "main1",
+  "executable":  "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12",
+  "cpu_count":  112,
+  "cpu_count_logical":  224,
+  "gpu":  "NVIDIA H200",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "1356758433792",
+      "used":  "257963536384"
+    }
+  },
+  "memory":  {
+    "total":  "2163961778176"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c"
+    },
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262"
+    },
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790"
+    },
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0"
+    }
+  ],
+  "cudaVersion":  "12.9",
+  "writerId":  "mf7qlulwtbceq6fkw52thj4sgf25e3dz"
+}

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/wandb/run-20260402_224531-spcd04xe/files/wandb-summary.json

@@ -0,0 +1 @@
+{"fake_score/local_total_params":[399300904,399021352,399021352,399021352],"optimizer/non_finite_grads_count (model_key net)":0,"_wandb":{"runtime":445},"optimizer/grad_norm (model_key net)":0,"_runtime":445.189429909,"fake_score/total_params":1596364960,"profiler/avg_forward_pass_time":16.967717550694942,"model/trainable_params":1596364960,"model/total_params":17311830496,"train/fake_score_loss":0.0072784423828125,"_timestamp":1.7751377931214402e+09,"profiler/avg_iteration_time":57.83046340942383,"profiler/data_loading_time":0.000791529193520546,"fake_score/local_trainable_params":[43745280,43745280,43745280,43745280],"profiler/backward_pass_time":11.342406308278441,"net/local_total_params":[376675360,376398880,376398880,376398880],"train/total_loss":0.0072784423828125,"model/local_total_params":[4350425968,4348630896,4348630896,4348630896],"_step":4,"optimizer/lr_fake_score":2e-06,"net/trainable_params":1421383840,"train/gan_loss_disc":0,"profiler/optimizer_step_time":1.187411269173026,"net/total_params":1421383840,"optimizer/lr_net":2e-06,"model/local_trainable_params":[420420640,420144160,420144160,420144160],"fake_score/trainable_params":174981120,"net/local_trainable_params":[376675360,376398880,376398880,376398880],"optimizer/iteration":4}

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/wandb/run-20260402_224531-spcd04xe/logs/debug-core.log

@@ -0,0 +1,15 @@
+{"time":"2026-04-02T22:45:31.195756192+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmphkxult0z/port-792541.txt","pid":792541,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-02T22:45:31.196239435+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":792541}
+{"time":"2026-04-02T22:45:31.196228389+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-792541-806747-3978962853/socket","Net":"unix"}}
+{"time":"2026-04-02T22:45:31.38167949+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-02T22:45:31.396742512+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"spcd04xe","id":"1(@)"}
+{"time":"2026-04-02T22:45:31.746828856+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"spcd04xe","id":"1(@)"}
+{"time":"2026-04-02T22:45:37.997230573+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ve1uhd81uimn"}
+{"time":"2026-04-02T22:52:57.910992536+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-04-02T22:52:57.911042628+09:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-04-02T22:52:57.911035318+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-04-02T22:52:57.911115888+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-04-02T22:52:57.911110682+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-792541-806747-3978962853/socket","Net":"unix"}}
+{"time":"2026-04-02T22:52:58.870224925+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-04-02T22:52:58.87024994+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-04-02T22:52:58.870261844+09:00","level":"INFO","msg":"server is closed"}

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/wandb/run-20260402_224531-spcd04xe/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-04-02T22:45:31.396833745+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
+{"time":"2026-04-02T22:45:31.746696076+09:00","level":"INFO","msg":"stream: created new stream","id":"spcd04xe"}
+{"time":"2026-04-02T22:45:31.746744779+09:00","level":"INFO","msg":"handler: started","stream_id":"spcd04xe"}
+{"time":"2026-04-02T22:45:31.746822827+09:00","level":"INFO","msg":"stream: started","id":"spcd04xe"}
+{"time":"2026-04-02T22:45:31.746841154+09:00","level":"INFO","msg":"sender: started","stream_id":"spcd04xe"}
+{"time":"2026-04-02T22:45:31.74684523+09:00","level":"INFO","msg":"writer: started","stream_id":"spcd04xe"}
+{"time":"2026-04-02T22:52:57.91103952+09:00","level":"INFO","msg":"stream: closing","id":"spcd04xe"}
+{"time":"2026-04-02T22:52:58.486184439+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-04-02T22:52:58.869727008+09:00","level":"INFO","msg":"handler: closed","stream_id":"spcd04xe"}
+{"time":"2026-04-02T22:52:58.869816672+09:00","level":"INFO","msg":"sender: closed","stream_id":"spcd04xe"}
+{"time":"2026-04-02T22:52:58.869827326+09:00","level":"INFO","msg":"stream: closed","id":"spcd04xe"}

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/wandb/run-20260402_224531-spcd04xe/logs/debug.log

@@ -0,0 +1,21 @@
+2026-04-02 22:45:31,146 INFO    MainThread:792541 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
+2026-04-02 22:45:31,146 INFO    MainThread:792541 [wandb_setup.py:_flush():81] Configure stats pid to 792541
+2026-04-02 22:45:31,146 INFO    MainThread:792541 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-02 22:45:31,146 INFO    MainThread:792541 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/wandb/run-20260402_224531-spcd04xe/logs/debug.log
+2026-04-02 22:45:31,146 INFO    MainThread:792541 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/wandb/run-20260402_224531-spcd04xe/logs/debug-internal.log
+2026-04-02 22:45:31,146 INFO    MainThread:792541 [wandb_init.py:init():844] calling init triggers
+2026-04-02 22:45:31,146 INFO    MainThread:792541 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf', 'name': 'sf_combined_step_test', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_lr1e5_10000iter_shift_5/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': "<class 'fastgen.callbacks.grad_clip.GradClipCallback'>"}, 'gpu_stats': {'every_n': '100', '_target_': "<class 'fastgen.callbacks.gpu_stats.GPUStatsCallback'>"}, 'train_profiler': {'every_n': '100', '_target_': "<class 'fastgen.callbacks.train_profiler.TrainProfilerCallback'>"}, 'param_count': {'_target_': "<class 'fastgen.callbacks.param_count.ParamCountCallback'>"}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': "<class 'fastgen.callbacks.ema.EMACallback'>"}, 'wandb': {'sample_logging_iter': '100', '_target_': "<class 'fastgen.callbacks.wandb.WandbCallback'>", 'fps': '25'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '20', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "<class 'fastgen.datasets.omniavatar_dataloader.OmniAvatarDataLoader'>"}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': '<function create_omniavatar_dataloader at 0x7ff2f56a3ce0>'}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "<class 'fastgen.networks.OmniAvatar.network_causal.CausalOmniAvatarWan'>"}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_phase2/step-10500.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "<class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>"}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt', 'merge_lora': 'False', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "<class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>"}, 'guidance_scale': '4.5', 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7ff2f5916660>'}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.937', '0.833', '0.624', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '4', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'bfloat16', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7ff2f5916660>'}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': "<class 'fastgen.networks.discriminators.Discriminator_EDM'>"}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7ff2f5916660>'}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None}, 'model_class': {'config': None, '_target_': "<class 'fastgen.methods.omniavatar_self_forcing.OmniAvatarSelfForcingModel'>"}, '_wandb': {}}
+2026-04-02 22:45:31,146 INFO    MainThread:792541 [wandb_init.py:init():892] starting backend
+2026-04-02 22:45:31,381 INFO    MainThread:792541 [wandb_init.py:init():895] sending inform_init request
+2026-04-02 22:45:31,394 INFO    MainThread:792541 [wandb_init.py:init():903] backend started and connected
+2026-04-02 22:45:31,398 INFO    MainThread:792541 [wandb_init.py:init():973] updated telemetry
+2026-04-02 22:45:31,413 INFO    MainThread:792541 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-04-02 22:45:32,719 INFO    MainThread:792541 [wandb_init.py:init():1042] starting run threads in backend
+2026-04-02 22:45:32,993 INFO    MainThread:792541 [wandb_run.py:_console_start():2524] atexit reg
+2026-04-02 22:45:32,993 INFO    MainThread:792541 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-04-02 22:45:32,994 INFO    MainThread:792541 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-04-02 22:45:32,994 INFO    MainThread:792541 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-04-02 22:45:32,998 INFO    MainThread:792541 [wandb_init.py:init():1082] run started, returning control to user process
+2026-04-02 22:52:57,911 INFO    wandb-AsyncioManager-main:792541 [service_client.py:_forward_responses():134] Reached EOF.
+2026-04-02 22:52:57,911 INFO    wandb-AsyncioManager-main:792541 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/wandb/run-20260402_224531-spcd04xe/run-spcd04xe.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c85a6826c3e4463f05f583260010422503fd8df994d4cb8222a7140a5f63297a
+size 132588

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_combined_step_test/wandb_id.txt

@@ -0,0 +1 @@
+spcd04xe

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_no_reqgrad_toggle/config.yaml

@@ -0,0 +1,259 @@
+dataloader_train:
+  _target_: <class 'fastgen.datasets.omniavatar_dataloader.OmniAvatarDataLoader'>
+  batch_size: '8'
+  data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt
+  latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png
+  neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt
+  num_workers: '2'
+  use_ref_sequence: 'True'
+dataloader_val:
+  _target_: <function create_omniavatar_dataloader at 0x7feb4c60be20>
+  batch_size: '1'
+  data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt
+  latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png
+  load_ode_path: 'False'
+  neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt
+  num_workers: '2'
+  use_ref_sequence: 'True'
+eval:
+  max_ckpt: '100000000'
+  min_ckpt: '0'
+  num_samples: '50000'
+  samples_dir: samples
+  save_images: 'False'
+log_config:
+  group: omniavatar_sf
+  name: sf_no_reqgrad_toggle
+  project: OmniAvatar-FastGen
+  wandb_credential: ./credentials/wandb_api.txt
+  wandb_entity: paulhcho
+  wandb_mode: disabled
+model:
+  add_teacher_to_fsdp_dict: 'True'
+  context_noise: '0.0'
+  ddp_find_unused_parameters: 'True'
+  device: cuda
+  discriminator:
+    _target_: <class 'fastgen.networks.discriminators.Discriminator_EDM'>
+    all_res:
+    - '32'
+    - '16'
+    - '8'
+    feature_indices: '{0, 1, 2}'
+    in_channels: '256'
+  discriminator_optimizer:
+    _target_: <function get_optimizer at 0x7feb4c86a660>
+    betas:
+    - '0.9'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: '0.0001'
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  discriminator_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  enable_gradient_in_rollout: 'True'
+  enable_preprocessors: 'True'
+  fake_score: null
+  fake_score_net:
+    _target_: <class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors
+    in_dim: '65'
+    merge_lora: 'False'
+    mode: v2v
+    model_size: 1.3B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt
+    schedule_type: rf
+    use_audio: 'True'
+  fake_score_optimizer:
+    _target_: <function get_optimizer at 0x7feb4c86a660>
+    betas:
+    - '0.0'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: 2e-06
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  fake_score_pred_type: x0
+  fake_score_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  fsdp_meta_init: 'False'
+  gan_loss_weight_gen: '0'
+  gan_r1_reg_alpha: '0.1'
+  gan_r1_reg_weight: '0.0'
+  gan_use_same_t_noise: 'False'
+  grad_scaler_enabled: 'False'
+  grad_scaler_growth_interval: '2000'
+  grad_scaler_init_scale: '65536.0'
+  guidance_scale: '4.5'
+  input_shape:
+  - '16'
+  - '21'
+  - '64'
+  - '64'
+  last_step_only: 'False'
+  load_student_weights: 'False'
+  net:
+    _target_: <class 'fastgen.networks.OmniAvatar.network_causal.CausalOmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors
+    chunk_size: '3'
+    in_dim: '65'
+    mode: v2v
+    model_size: 1.3B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt
+    schedule_type: rf
+    total_num_frames: '21'
+    use_audio: 'True'
+  net_optimizer:
+    _target_: <function get_optimizer at 0x7feb4c86a660>
+    betas:
+    - '0.0'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: 2e-06
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  net_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  precision: bfloat16
+  precision_amp: null
+  precision_amp_enc: null
+  precision_amp_infer: null
+  precision_fsdp: bfloat16
+  pretrained_model_path: ''
+  pretrained_student_net_path: ''
+  same_step_across_blocks: 'True'
+  sample_t_cfg:
+    log_t_df: '0.01'
+    max_t: '0.999'
+    min_t: '0.001'
+    shift: '5.0'
+    t_list:
+    - '0.999'
+    - '0.937'
+    - '0.833'
+    - '0.624'
+    - '0.0'
+    time_dist_type: shifted
+    train_p_mean: '-1.1'
+    train_p_std: '2.0'
+  skip_layers: null
+  start_gradient_frame: '0'
+  student_sample_steps: '4'
+  student_sample_type: sde
+  student_update_freq: '5'
+  teacher:
+    _target_: <class 'fastgen.networks.OmniAvatar.network.OmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors
+    in_dim: '65'
+    merge_lora: 'True'
+    mode: v2v
+    model_size: 14B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_phase2/step-10500.pt
+    schedule_type: rf
+    use_audio: 'True'
+  use_ema: 'False'
+model_class:
+  _target_: <class 'fastgen.methods.omniavatar_self_forcing.OmniAvatarSelfForcingModel'>
+  config: null
+trainer:
+  augment_pipe: null
+  batch_size_global: null
+  callbacks:
+    ema:
+      _target_: <class 'fastgen.callbacks.ema.EMACallback'>
+      beta: '0.9999'
+      ema_halflife_kimg: '500'
+      ema_rampup_ratio: '0.05'
+      gamma: '16.97'
+      start_iter: '0'
+      type: constant
+    gpu_stats:
+      _target_: <class 'fastgen.callbacks.gpu_stats.GPUStatsCallback'>
+      every_n: '100'
+    grad_clip:
+      _target_: <class 'fastgen.callbacks.grad_clip.GradClipCallback'>
+      grad_norm: '10.0'
+      model_key: net
+    param_count:
+      _target_: <class 'fastgen.callbacks.param_count.ParamCountCallback'>
+    train_profiler:
+      _target_: <class 'fastgen.callbacks.train_profiler.TrainProfilerCallback'>
+      every_n: '100'
+    wandb:
+      _target_: <class 'fastgen.callbacks.wandb.WandbCallback'>
+      fps: '25'
+      sample_logging_iter: '100'
+  checkpointer:
+    pretrained_ckpt_key_map:
+      net: net
+    pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_lr1e5_10000iter_shift_5/checkpoints/0005000.pth
+    s3_container: s3://checkpoints/fastgen
+    s3_credential: ./credentials/s3.json
+    save_dir: /tmp/FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_no_reqgrad_toggle/checkpoints
+    use_s3: 'False'
+  cudnn:
+    benchmark: 'True'
+    deterministic: 'False'
+  ddp: 'False'
+  fsdp: 'True'
+  fsdp_cpu_offload: 'False'
+  fsdp_min_num_params: '10000000'
+  fsdp_sharding_group_size: null
+  global_vars: null
+  global_vars_val:
+  - null
+  grad_accum_rounds: '2'
+  logging_iter: '1'
+  max_iter: '10'
+  offload_module_in_decoding: 'False'
+  resume: 'False'
+  save_ckpt_iter: '100'
+  seed: '0'
+  skip_initial_validation: 'True'
+  tf32_enabled: 'True'
+  val_seed: null
+  validation_iter: '100'
+  visualize_teacher: 'False'

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_sink1_window7_redmd_syncc_beta0p25/checkpoints/0000100.net_model/.metadata

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68cfb6be85ca8bb4cb2c99e580d579759b315332fe04ff9583e8a17503710b70
+size 614328

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_sink1_window7_redmd_syncc_beta0p25/checkpoints/0000100.net_model/__0_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58d137502c435eeaa445c868ec414ada748effb592f340238578f82a40be7a9c
+size 1424446005

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_sink1_window7_redmd_syncc_beta0p25/checkpoints/0000100.net_model/__1_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d60a72b6714edac135314634cae333779c2cdc040ac2bae1b34bfe521981579d
+size 1401881227

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_sink1_window7_redmd_syncc_beta0p25/checkpoints/0000100.net_model/__2_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c33b96f7c9d96d1b61c2817be29f21a1fffbc41eef2ba6d7e4d7d4bb3eb2543
+size 1406759189

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_sink1_window7_redmd_syncc_beta0p25/checkpoints/0000100.net_model/__3_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c16e8089a70fb63a6ef5e83f93b2bc5ef002d78b3e8f1c61ae1fd9386e18db0b
+size 1401210558

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_sink1_window7_redmd_syncc_beta0p25/checkpoints/0000100.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c9e7bf7b483afea9b5f5fc9f6d1a8368eea2246e5e5404c08b9fba8acdc0064
+size 1901

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_sink1_window7_redmd_syncc_beta0p25/checkpoints/0000200.net_model/.metadata

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6aa773eb266887bcf43277c27dddf54d6f73581fb8a49aefd509ff3fd7699d8b
+size 614328

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_sink1_window7_redmd_syncc_beta0p25/checkpoints/0000200.net_model/__0_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f79a6720f3855aca52a768cc9b64766d16fbc6af95c459f3d236fee4056a8b2
+size 1424446005

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_sink1_window7_redmd_syncc_beta0p25/checkpoints/0000200.net_model/__1_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4e4b4d3a27464533ae9b16fe7eb25319df4483379605ce23676ce66ee538e9c
+size 1401881227

FASTGEN_SF_OUTPUT/OmniAvatar-FastGen/omniavatar_sf/sf_sink1_window7_redmd_syncc_beta0p25/checkpoints/0000200.net_model/__2_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c6fafdbb008d602689cc51bdb7811e31fdab406e70859f6f647970016bfde2a
+size 1406759189