Backup: DF_LORA_T769_8GPU_BS8

.gitattributes

@@ -1897,3 +1897,14 @@ hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_4STEP_NO_REWARD/OmniAvatar-FastGen/
 hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_4STEP_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_4step_no_reward/wandb/run-20260503_124619-71vuvjwo/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text
 hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_4STEP_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_4step_no_reward/wandb/run-20260503_124619-71vuvjwo/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text
 hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_4STEP_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_4step_no_reward/wandb/run-20260503_124619-71vuvjwo/run-71vuvjwo.wandb filter=lfs diff=lfs merge=lfs -text
+DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_122daebb61ef6826b8de.mp4 filter=lfs diff=lfs merge=lfs -text
+DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_479372da7ed9e768cc22.mp4 filter=lfs diff=lfs merge=lfs -text
+DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_509f08fac994ab8cebb6.mp4 filter=lfs diff=lfs merge=lfs -text
+DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_5d1790cb2d9c8554b75b.mp4 filter=lfs diff=lfs merge=lfs -text
+DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_81b3c24ab2930c8fd6c1.mp4 filter=lfs diff=lfs merge=lfs -text
+DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_8495e04002c0da67769b.mp4 filter=lfs diff=lfs merge=lfs -text
+DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_88d3eab6ac986b51e272.mp4 filter=lfs diff=lfs merge=lfs -text
+DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_8bcc788bab53b0c99f6b.mp4 filter=lfs diff=lfs merge=lfs -text
+DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_962d9fe735e704d64274.mp4 filter=lfs diff=lfs merge=lfs -text
+DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_b38481beb241c6eca7c1.mp4 filter=lfs diff=lfs merge=lfs -text
+DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/run-aogal2qe.wandb filter=lfs diff=lfs merge=lfs -text

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/config.yaml

@@ -0,0 +1,192 @@
+dataloader_train:
+  _target_: <class 'fastgen.datasets.omniavatar_dataloader.OmniAvatarDataLoader'>
+  batch_size: '8'
+  data_list_path: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/v2v_training_data/video_square_path.txt
+  latentsync_mask_path: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/mask.png
+  load_ode_path: 'False'
+  neg_text_emb_path: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/text_emb/neg_text_emb.pt
+  num_workers: '4'
+  use_ref_sequence: 'True'
+dataloader_val:
+  _target_: <function create_omniavatar_dataloader at 0x7f9e8dc118a0>
+  batch_size: '1'
+  data_list_path: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/v2v_training_data/video_square_val10.txt
+  latentsync_mask_path: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/mask.png
+  load_ode_path: 'False'
+  neg_text_emb_path: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/text_emb/neg_text_emb.pt
+  num_workers: '2'
+  use_ref_sequence: 'True'
+eval:
+  max_ckpt: '100000000'
+  min_ckpt: '0'
+  num_samples: '50000'
+  samples_dir: samples
+  save_images: 'False'
+log_config:
+  group: omniavatar_df_audiofix
+  name: df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter
+  project: OmniAvatar-FastGen
+  wandb_credential: ./credentials/wandb_api.txt
+  wandb_entity: paulhcho
+  wandb_mode: online
+model:
+  add_teacher_to_fsdp_dict: 'True'
+  context_noise: '0.0'
+  ddp_find_unused_parameters: 'True'
+  device: cuda
+  enable_preprocessors: 'True'
+  fake_score_net: null
+  fsdp_meta_init: 'False'
+  grad_scaler_enabled: 'False'
+  grad_scaler_growth_interval: '2000'
+  grad_scaler_init_scale: '65536.0'
+  guidance_scale: null
+  input_shape:
+  - '16'
+  - '21'
+  - '64'
+  - '64'
+  load_student_weights: 'True'
+  net:
+    _target_: <class 'fastgen.networks.OmniAvatar.network_causal.CausalOmniAvatarWan'>
+    audio_hidden_size: '32'
+    base_model_paths: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/OmniAvatar-Train/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors
+    chunk_size: '3'
+    in_dim: '65'
+    lora_alpha: '64'
+    lora_rank: '128'
+    merge_lora: 'False'
+    mode: v2v
+    model_size: 1.3B
+    net_pred_type: flow
+    omniavatar_ckpt_path: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data_checkpoints/1.3B-step-1000.pt
+    schedule_type: rf
+    stochastic_attn_configs:
+    - local_attn_size: '7'
+      sink_size: '1'
+      weight: '0.2'
+    - local_attn_size: '10'
+      sink_size: '1'
+      weight: '0.2'
+    - local_attn_size: '13'
+      sink_size: '1'
+      weight: '0.2'
+    - local_attn_size: '9'
+      sink_size: '3'
+      weight: '0.2'
+    - local_attn_size: '12'
+      sink_size: '3'
+      weight: '0.2'
+    total_num_frames: '21'
+    unfreeze_modules:
+    - _core.audio_proj
+    - _core.audio_cond_projs
+    - _core.patch_embedding
+    use_audio: 'True'
+    use_dynamic_rope: 'False'
+  net_optimizer:
+    _target_: <function get_optimizer at 0x7f9e90111940>
+    betas:
+    - '0.9'
+    - '0.999'
+    eps: 1e-08
+    fused: 'False'
+    lr: 1e-05
+    model: null
+    optim_type: adamw
+    weight_decay: '0.01'
+  net_scheduler:
+    _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+    cycle_lengths:
+    - '10000000000'
+    f_max:
+    - '1.0'
+    f_min:
+    - '1.0'
+    f_start:
+    - 1e-06
+    warm_up_steps:
+    - '0'
+  precision: bfloat16
+  precision_amp: null
+  precision_amp_enc: null
+  precision_amp_infer: null
+  precision_fsdp: float32
+  pretrained_model_path: ''
+  pretrained_student_net_path: ''
+  sample_t_cfg:
+    log_t_df: '0.01'
+    max_t: '0.999'
+    min_t: '0.001'
+    shift: '5.0'
+    t_list:
+    - '0.999'
+    - '0.769'
+    - '0.0'
+    time_dist_type: shifted
+    train_p_mean: '-1.1'
+    train_p_std: '2.0'
+  skip_layers: null
+  student_sample_steps: '2'
+  student_sample_type: sde
+  teacher: null
+  timestep_cfg:
+    enabled: 'False'
+    t_hi: '1.0'
+    t_lo: '0.0'
+  use_ema: 'False'
+  vae_path: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/OmniAvatar-Train/pretrained_models/Wan2.1-T2V-1.3B/Wan2.1_VAE.pth
+model_class:
+  _target_: <class 'fastgen.methods.omniavatar_diffusion_forcing.OmniAvatarDiffusionForcingModel'>
+  config: null
+trainer:
+  augment_pipe: null
+  batch_size_global: null
+  callbacks:
+    gpu_stats:
+      _target_: <class 'fastgen.callbacks.gpu_stats.GPUStatsCallback'>
+      every_n: '100'
+    grad_clip:
+      _target_: <class 'fastgen.callbacks.grad_clip.GradClipCallback'>
+      grad_norm: '10.0'
+      model_key: net
+    param_count:
+      _target_: <class 'fastgen.callbacks.param_count.ParamCountCallback'>
+    train_profiler:
+      _target_: <class 'fastgen.callbacks.train_profiler.TrainProfilerCallback'>
+      every_n: '100'
+    wandb:
+      _target_: <class 'fastgen.callbacks.wandb.WandbCallback'>
+      fps: '25'
+      sample_logging_iter: '500'
+  checkpointer:
+    pretrained_ckpt_key_map:
+      net: net
+    pretrained_ckpt_path: ''
+    s3_container: s3://checkpoints/fastgen
+    s3_credential: ./credentials/s3.json
+    save_dir: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/output/DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/checkpoints
+    use_s3: 'False'
+  cudnn:
+    benchmark: 'True'
+    deterministic: 'False'
+  ddp: 'True'
+  fsdp: 'False'
+  fsdp_cpu_offload: 'False'
+  fsdp_min_num_params: '10000000'
+  fsdp_sharding_group_size: null
+  global_vars: null
+  global_vars_val:
+  - null
+  grad_accum_rounds: '1'
+  logging_iter: '1'
+  max_iter: '5000'
+  offload_module_in_decoding: 'False'
+  resume: 'False'
+  save_ckpt_iter: '500'
+  seed: '0'
+  skip_initial_validation: 'True'
+  tf32_enabled: 'True'
+  val_seed: null
+  validation_iter: '500'
+  visualize_teacher: 'False'

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/debug-internal.log

@@ -0,0 +1,145 @@
+{"time":"2026-04-30T23:05:47.717661886+09:00","level":"INFO","msg":"wandb-core"}
+{"time":"2026-04-30T23:05:47.719875838+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.1"}
+{"time":"2026-04-30T23:05:48.202102105+09:00","level":"INFO","msg":"stream: created new stream","id":"aogal2qe"}
+{"time":"2026-04-30T23:05:48.202176404+09:00","level":"INFO","msg":"handler: started"}
+{"time":"2026-04-30T23:05:48.203751244+09:00","level":"INFO","msg":"stream: started"}
+{"time":"2026-04-30T23:05:48.203784235+09:00","level":"INFO","msg":"sender: started"}
+{"time":"2026-04-30T23:05:48.203780548+09:00","level":"INFO","msg":"writer: started","stream_id":"aogal2qe"}
+{"time":"2026-04-30T23:05:49.289209394+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-04-30T23:05:49.628819533+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:06:04.28911266+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":19,"uploaded_len":2}
+{"time":"2026-04-30T23:06:04.68488743+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:06:19.28947862+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-04-30T23:06:19.593106092+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:06:34.289531575+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
+{"time":"2026-04-30T23:06:34.638885012+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:06:49.289408032+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":6,"events_lines":2,"console_offset":19,"console_lines":1}
+{"time":"2026-04-30T23:06:49.55441404+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:07:04.2893822+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
+{"time":"2026-04-30T23:07:04.547174503+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:07:19.289556072+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":10,"events_lines":2}
+{"time":"2026-04-30T23:07:19.542062737+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:07:34.289675411+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":12,"events_lines":2,"console_offset":20,"console_lines":29}
+{"time":"2026-04-30T23:07:34.590652485+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:07:49.289694092+09:00","level":"INFO","msg":"filestream: sending request","total_files":3,"events_offset":14,"events_lines":2,"console_offset":49,"console_lines":9,"uploaded_len":10}
+{"time":"2026-04-30T23:07:49.578907428+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:08:04.289321845+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":16,"events_lines":2,"console_offset":58,"console_lines":3}
+{"time":"2026-04-30T23:08:04.654848542+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:08:19.302053487+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":18,"events_lines":2,"console_offset":61,"console_lines":10}
+{"time":"2026-04-30T23:08:19.606327435+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:08:34.288993064+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":20,"events_lines":2}
+{"time":"2026-04-30T23:08:34.57863+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:08:49.289673912+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":22,"events_lines":2}
+{"time":"2026-04-30T23:08:49.597082041+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:09:04.289440794+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":24,"events_lines":2}
+{"time":"2026-04-30T23:09:04.62933334+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:09:19.289512374+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":26,"events_lines":2}
+{"time":"2026-04-30T23:09:19.579604059+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:09:34.288965379+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":28,"events_lines":2}
+{"time":"2026-04-30T23:09:34.594544765+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:09:49.288782859+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
+{"time":"2026-04-30T23:09:49.612205294+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:10:04.289135035+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":32,"events_lines":2}
+{"time":"2026-04-30T23:10:04.630551549+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:10:19.288780712+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":34,"events_lines":2}
+{"time":"2026-04-30T23:10:19.682644618+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:10:34.289481389+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":36,"events_lines":2}
+{"time":"2026-04-30T23:10:34.647039103+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:10:49.289265985+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":38,"events_lines":2}
+{"time":"2026-04-30T23:10:49.730743901+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:11:04.289172039+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":40,"events_lines":2}
+{"time":"2026-04-30T23:11:04.731077132+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:11:19.289448884+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":42,"events_lines":2}
+{"time":"2026-04-30T23:11:19.667453476+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:11:34.289592911+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":44,"events_lines":2}
+{"time":"2026-04-30T23:11:34.668128144+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:11:49.288778693+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":46,"events_lines":2}
+{"time":"2026-04-30T23:11:49.644035721+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:12:04.289349786+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":48,"events_lines":2}
+{"time":"2026-04-30T23:12:04.736082536+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:12:19.288945368+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":50,"events_lines":2}
+{"time":"2026-04-30T23:12:19.610329497+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:12:34.289571483+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":52,"events_lines":2}
+{"time":"2026-04-30T23:12:34.587850694+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:12:49.288764613+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":54,"events_lines":2}
+{"time":"2026-04-30T23:12:49.719622981+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:13:04.288637224+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":56,"events_lines":2}
+{"time":"2026-04-30T23:13:04.582812338+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:13:19.289301406+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":58,"events_lines":2}
+{"time":"2026-04-30T23:13:19.553204842+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:13:34.289451588+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":60,"events_lines":2}
+{"time":"2026-04-30T23:13:34.534556354+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:13:49.288622643+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":62,"events_lines":2}
+{"time":"2026-04-30T23:13:49.595316475+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:14:04.289144412+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":64,"events_lines":2}
+{"time":"2026-04-30T23:14:04.543244073+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:14:19.288823516+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":66,"events_lines":2}
+{"time":"2026-04-30T23:14:19.571833716+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:14:34.289509199+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":68,"events_lines":2}
+{"time":"2026-04-30T23:14:34.579585848+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:14:49.288824765+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":70,"events_lines":2}
+{"time":"2026-04-30T23:14:49.612924595+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:15:04.2892304+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":72,"events_lines":2}
+{"time":"2026-04-30T23:15:04.566368977+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:15:19.288719459+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":74,"events_lines":2}
+{"time":"2026-04-30T23:15:19.609636252+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:15:34.289310014+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":76,"events_lines":2}
+{"time":"2026-04-30T23:15:34.584427145+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:15:49.288956977+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":78,"events_lines":2}
+{"time":"2026-04-30T23:15:49.614829139+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:16:04.289218328+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":80,"events_lines":2}
+{"time":"2026-04-30T23:16:04.60993107+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:16:19.289171247+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":82,"events_lines":2,"console_offset":71,"console_lines":3}
+{"time":"2026-04-30T23:16:19.628814484+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:16:34.288676582+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":84,"events_lines":2}
+{"time":"2026-04-30T23:16:34.689081134+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:16:49.288933188+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":86,"events_lines":2}
+{"time":"2026-04-30T23:16:49.62914172+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:17:04.288607448+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":88,"events_lines":2}
+{"time":"2026-04-30T23:17:04.589594346+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:17:19.289321381+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":90,"events_lines":2}
+{"time":"2026-04-30T23:17:19.564264059+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:17:34.288994724+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":92,"events_lines":2}
+{"time":"2026-04-30T23:17:34.552863001+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:17:49.289503425+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":94,"events_lines":2}
+{"time":"2026-04-30T23:17:49.588243139+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:18:04.28878216+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":96,"events_lines":2}
+{"time":"2026-04-30T23:18:04.554107868+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:18:19.289179022+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":98,"events_lines":2}
+{"time":"2026-04-30T23:18:19.612210314+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:18:34.288875601+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":100,"events_lines":2}
+{"time":"2026-04-30T23:18:34.581426522+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:18:49.289382733+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":102,"events_lines":2}
+{"time":"2026-04-30T23:18:49.571645992+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:19:04.28863327+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":104,"events_lines":2}
+{"time":"2026-04-30T23:19:04.567218404+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:19:19.288592986+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":106,"events_lines":2}
+{"time":"2026-04-30T23:19:19.559133381+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:19:34.288571933+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":108,"events_lines":2}
+{"time":"2026-04-30T23:19:34.543783797+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:19:49.289279029+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":110,"events_lines":2}
+{"time":"2026-04-30T23:19:49.634365885+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:20:04.288850186+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":112,"events_lines":2}
+{"time":"2026-04-30T23:20:04.569464453+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:20:19.289155372+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":114,"events_lines":2}
+{"time":"2026-04-30T23:20:19.613994616+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:20:34.289328425+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":116,"events_lines":2}
+{"time":"2026-04-30T23:20:34.620516915+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:20:49.288670175+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":118,"events_lines":2}
+{"time":"2026-04-30T23:20:49.701752992+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:21:04.289448285+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":120,"events_lines":2}
+{"time":"2026-04-30T23:21:04.672086777+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:21:19.288957688+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":122,"events_lines":2}
+{"time":"2026-04-30T23:21:19.808028855+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:21:34.289496467+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":124,"events_lines":2}
+{"time":"2026-04-30T23:21:34.598270838+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:21:49.288582737+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":126,"events_lines":2}
+{"time":"2026-04-30T23:21:49.602869427+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:22:04.289027983+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":128,"events_lines":2}
+{"time":"2026-04-30T23:22:04.602488321+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:22:19.288843665+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":130,"events_lines":2}
+{"time":"2026-04-30T23:22:19.590625944+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:22:34.288973354+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":132,"events_lines":2}
+{"time":"2026-04-30T23:22:34.576595162+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:22:49.28941316+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":134,"events_lines":1}
+{"time":"2026-04-30T23:22:49.617243724+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/debug.log

@@ -0,0 +1,19 @@
+2026-04-30 23:05:47,480 INFO    MainThread:2767876 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
+2026-04-30 23:05:47,480 INFO    MainThread:2767876 [wandb_setup.py:_flush():81] Configure stats pid to 2767876
+2026-04-30 23:05:47,480 INFO    MainThread:2767876 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-30 23:05:47,480 INFO    MainThread:2767876 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/output/DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/logs/debug.log
+2026-04-30 23:05:47,480 INFO    MainThread:2767876 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/output/DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/logs/debug-internal.log
+2026-04-30 23:05:47,480 INFO    MainThread:2767876 [wandb_init.py:init():844] calling init triggers
+2026-04-30 23:05:47,480 INFO    MainThread:2767876 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_df_audiofix', 'name': 'df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/output/DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': "<class 'fastgen.callbacks.grad_clip.GradClipCallback'>"}, 'gpu_stats': {'every_n': '100', '_target_': "<class 'fastgen.callbacks.gpu_stats.GPUStatsCallback'>"}, 'train_profiler': {'every_n': '100', '_target_': "<class 'fastgen.callbacks.train_profiler.TrainProfilerCallback'>"}, 'param_count': {'_target_': "<class 'fastgen.callbacks.param_count.ParamCountCallback'>"}, 'wandb': {'sample_logging_iter': '500', 'fps': '25', '_target_': "<class 'fastgen.callbacks.wandb.WandbCallback'>"}}, 'save_ckpt_iter': '500', 'validation_iter': '500', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '5000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'True', 'fsdp': 'False', 'tf32_enabled': 'True', 'grad_accum_rounds': '1', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/mask.png', 'batch_size': '8', 'num_workers': '4', 'neg_text_emb_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/text_emb/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': "<class 'fastgen.datasets.omniavatar_dataloader.OmniAvatarDataLoader'>"}, 'dataloader_val': {'data_list_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/text_emb/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': '<function create_omniavatar_dataloader at 0x7fa4931818a0>'}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/OmniAvatar-Train/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data_checkpoints/1.3B-step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'use_dynamic_rope': 'False', 'stochastic_attn_configs': [{'local_attn_size': '7', 'sink_size': '1', 'weight': '0.2'}, {'local_attn_size': '10', 'sink_size': '1', 'weight': '0.2'}, {'local_attn_size': '13', 'sink_size': '1', 'weight': '0.2'}, {'local_attn_size': '9', 'sink_size': '3', 'weight': '0.2'}, {'local_attn_size': '12', 'sink_size': '3', 'weight': '0.2'}], '_target_': "<class 'fastgen.networks.OmniAvatar.network_causal.CausalOmniAvatarWan'>", 'merge_lora': 'False', 'unfreeze_modules': ['_core.audio_proj', '_core.audio_cond_projs', '_core.patch_embedding'], 'lora_rank': '128', 'lora_alpha': '64'}, 'teacher': None, 'fake_score_net': None, 'guidance_scale': None, 'timestep_cfg': {'enabled': 'False', 't_lo': '0.0', 't_hi': '1.0'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '1e-05', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7fa499799940>'}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'True', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'context_noise': '0.0', 'vae_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/OmniAvatar-Train/pretrained_models/Wan2.1-T2V-1.3B/Wan2.1_VAE.pth'}, 'model_class': {'config': None, '_target_': "<class 'fastgen.methods.omniavatar_diffusion_forcing.OmniAvatarDiffusionForcingModel'>"}, '_wandb': {}}
+2026-04-30 23:05:47,480 INFO    MainThread:2767876 [wandb_init.py:init():892] starting backend
+2026-04-30 23:05:47,705 INFO    MainThread:2767876 [wandb_init.py:init():895] sending inform_init request
+2026-04-30 23:05:47,714 INFO    MainThread:2767876 [wandb_init.py:init():903] backend started and connected
+2026-04-30 23:05:47,716 INFO    MainThread:2767876 [wandb_init.py:init():973] updated telemetry
+2026-04-30 23:05:47,743 INFO    MainThread:2767876 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-04-30 23:05:48,972 INFO    MainThread:2767876 [wandb_init.py:init():1042] starting run threads in backend
+2026-04-30 23:05:49,282 INFO    MainThread:2767876 [wandb_run.py:_console_start():2524] atexit reg
+2026-04-30 23:05:49,283 INFO    MainThread:2767876 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-04-30 23:05:49,283 INFO    MainThread:2767876 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-04-30 23:05:49,283 INFO    MainThread:2767876 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-04-30 23:05:49,286 INFO    MainThread:2767876 [wandb_init.py:init():1082] run started, returning control to user process

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_225926-w9xbjand/files/config.yaml

@@ -0,0 +1,318 @@
+_wandb:
+    value:
+        cli_version: 0.25.1
+        e:
+            7agpz9rvil27ulbaooy9tu3j71z6whib:
+                args:
+                    - --config=fastgen/configs/experiments/OmniAvatar/config_df_shift_5_lora_t769.py
+                    - '-'
+                    - dataloader_train.batch_size=8
+                    - trainer.ddp=True
+                    - trainer.max_iter=5000
+                    - trainer.save_ckpt_iter=500
+                    - trainer.resume=False
+                    - log_config.group=omniavatar_df_audiofix
+                    - log_config.name=df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter
+                    - log_config.project=OmniAvatar-FastGen
+                    - log_config.wandb_entity=paulhcho
+                codePath: train.py
+                codePathLocal: train.py
+                cpu_count: 64
+                cpu_count_logical: 128
+                cudaVersion: "12.9"
+                disk:
+                    /:
+                        total: "1599530123264"
+                        used: "753074597888"
+                email: paul.hyunbin@gmail.com
+                executable: /usr/bin/python
+                git:
+                    commit: f5897815a4bdcc8f11212b24f367a6839a5409ca
+                    remote: https://github.com/paulcho98/FastGen.git
+                gpu: NVIDIA A100-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-c1619da3-e2c1-15b0-cb55-50e24b1f8e99
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-29f274f2-7065-4653-dcd1-055518ba099c
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-47a2dd51-a5dc-1d24-4713-ea97b7049d18
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-ad40d9cb-0070-8a6c-f0a3-d8d40d8233a7
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-97413b12-6c45-b8cd-a06a-886dd3bc94d6
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-060cf5c3-ba18-6ff1-9d4d-9e9c71bf6c67
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-6562d339-d2bb-95e2-3881-199a64c50ba7
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-a6834410-f241-7270-f5e3-129ecdc2e8f3
+                host: instance-35250
+                memory:
+                    total: "1081693446144"
+                os: Linux-5.15.0-142-generic-x86_64-with-glibc2.39
+                program: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/train.py
+                python: CPython 3.12.3
+                root: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/output/DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter
+                startedAt: "2026-04-30T13:59:26.861268Z"
+                writerId: 7agpz9rvil27ulbaooy9tu3j71z6whib
+        m: []
+        python_version: 3.12.3
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 50
+                - 53
+                - 63
+                - 71
+                - 83
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 50
+                - 53
+                - 63
+                - 71
+                - 83
+                - 98
+            "3":
+                - 13
+                - 14
+                - 16
+            "4": 3.12.3
+            "5": 0.25.1
+            "6": 5.3.0
+            "12": 0.25.1
+            "13": linux-x86_64
+dataloader_train:
+    value:
+        _target_: <class 'fastgen.datasets.omniavatar_dataloader.OmniAvatarDataLoader'>
+        batch_size: "8"
+        data_list_path: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/v2v_training_data/video_square_path.txt
+        latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png
+        load_ode_path: "False"
+        neg_text_emb_path: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/text_emb/neg_text_emb.pt
+        num_workers: "4"
+        use_ref_sequence: "True"
+dataloader_val:
+    value:
+        _target_: <function create_omniavatar_dataloader at 0x7f7b29e018a0>
+        batch_size: "1"
+        data_list_path: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/v2v_training_data/video_square_val10.txt
+        latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png
+        load_ode_path: "False"
+        neg_text_emb_path: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/text_emb/neg_text_emb.pt
+        num_workers: "2"
+        use_ref_sequence: "True"
+eval:
+    value:
+        max_ckpt: "100000000"
+        min_ckpt: "0"
+        num_samples: "50000"
+        samples_dir: samples
+        save_images: "False"
+log_config:
+    value:
+        group: omniavatar_df_audiofix
+        name: df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter
+        project: OmniAvatar-FastGen
+        wandb_credential: ./credentials/wandb_api.txt
+        wandb_entity: paulhcho
+        wandb_mode: online
+model:
+    value:
+        add_teacher_to_fsdp_dict: "True"
+        context_noise: "0.0"
+        ddp_find_unused_parameters: "True"
+        device: cuda
+        enable_preprocessors: "True"
+        fake_score_net: null
+        fsdp_meta_init: "False"
+        grad_scaler_enabled: "False"
+        grad_scaler_growth_interval: "2000"
+        grad_scaler_init_scale: "65536.0"
+        guidance_scale: null
+        input_shape:
+            - "16"
+            - "21"
+            - "64"
+            - "64"
+        load_student_weights: "True"
+        net:
+            _target_: <class 'fastgen.networks.OmniAvatar.network_causal.CausalOmniAvatarWan'>
+            audio_hidden_size: "32"
+            base_model_paths: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/OmniAvatar-Train/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors
+            chunk_size: "3"
+            in_dim: "65"
+            lora_alpha: "64"
+            lora_rank: "128"
+            merge_lora: "False"
+            mode: v2v
+            model_size: 1.3B
+            net_pred_type: flow
+            omniavatar_ckpt_path: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data_checkpoints/1.3B-step-1000.pt
+            schedule_type: rf
+            stochastic_attn_configs:
+                - local_attn_size: "7"
+                  sink_size: "1"
+                  weight: "0.2"
+                - local_attn_size: "10"
+                  sink_size: "1"
+                  weight: "0.2"
+                - local_attn_size: "13"
+                  sink_size: "1"
+                  weight: "0.2"
+                - local_attn_size: "9"
+                  sink_size: "3"
+                  weight: "0.2"
+                - local_attn_size: "12"
+                  sink_size: "3"
+                  weight: "0.2"
+            total_num_frames: "21"
+            unfreeze_modules:
+                - _core.audio_proj
+                - _core.audio_cond_projs
+                - _core.patch_embedding
+            use_audio: "True"
+            use_dynamic_rope: "False"
+        net_optimizer:
+            _target_: <function get_optimizer at 0x7f7b2c301940>
+            betas:
+                - "0.9"
+                - "0.999"
+            eps: "1e-08"
+            fused: "False"
+            lr: "1e-05"
+            model: null
+            optim_type: adamw
+            weight_decay: "0.01"
+        net_scheduler:
+            _target_: <class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>
+            cycle_lengths:
+                - "10000000000"
+            f_max:
+                - "1.0"
+            f_min:
+                - "1.0"
+            f_start:
+                - "1e-06"
+            warm_up_steps:
+                - "0"
+        precision: bfloat16
+        precision_amp: null
+        precision_amp_enc: null
+        precision_amp_infer: null
+        precision_fsdp: float32
+        pretrained_model_path: ""
+        pretrained_student_net_path: ""
+        sample_t_cfg:
+            log_t_df: "0.01"
+            max_t: "0.999"
+            min_t: "0.001"
+            shift: "5.0"
+            t_list:
+                - "0.999"
+                - "0.769"
+                - "0.0"
+            time_dist_type: shifted
+            train_p_mean: "-1.1"
+            train_p_std: "2.0"
+        skip_layers: null
+        student_sample_steps: "2"
+        student_sample_type: sde
+        teacher: null
+        timestep_cfg:
+            enabled: "False"
+            t_hi: "1.0"
+            t_lo: "0.0"
+        use_ema: "False"
+        vae_path: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/OmniAvatar-Train/pretrained_models/Wan2.1-T2V-1.3B/Wan2.1_VAE.pth
+model_class:
+    value:
+        _target_: <class 'fastgen.methods.omniavatar_diffusion_forcing.OmniAvatarDiffusionForcingModel'>
+        config: null
+trainer:
+    value:
+        augment_pipe: null
+        batch_size_global: null
+        callbacks:
+            gpu_stats:
+                _target_: <class 'fastgen.callbacks.gpu_stats.GPUStatsCallback'>
+                every_n: "100"
+            grad_clip:
+                _target_: <class 'fastgen.callbacks.grad_clip.GradClipCallback'>
+                grad_norm: "10.0"
+                model_key: net
+            param_count:
+                _target_: <class 'fastgen.callbacks.param_count.ParamCountCallback'>
+            train_profiler:
+                _target_: <class 'fastgen.callbacks.train_profiler.TrainProfilerCallback'>
+                every_n: "100"
+            wandb:
+                _target_: <class 'fastgen.callbacks.wandb.WandbCallback'>
+                fps: "25"
+                sample_logging_iter: "500"
+        checkpointer:
+            pretrained_ckpt_key_map:
+                net: net
+            pretrained_ckpt_path: ""
+            s3_container: s3://checkpoints/fastgen
+            s3_credential: ./credentials/s3.json
+            save_dir: /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/output/DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/checkpoints
+            use_s3: "False"
+        cudnn:
+            benchmark: "True"
+            deterministic: "False"
+        ddp: "True"
+        fsdp: "False"
+        fsdp_cpu_offload: "False"
+        fsdp_min_num_params: "10000000"
+        fsdp_sharding_group_size: null
+        global_vars: null
+        global_vars_val:
+            - null
+        grad_accum_rounds: "1"
+        logging_iter: "1"
+        max_iter: "5000"
+        offload_module_in_decoding: "False"
+        resume: "False"
+        save_ckpt_iter: "500"
+        seed: "0"
+        skip_initial_validation: "True"
+        tf32_enabled: "True"
+        val_seed: null
+        validation_iter: "500"
+        visualize_teacher: "False"

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_225926-w9xbjand/files/output.log

@@ -0,0 +1,61 @@
+[Apr 30, 2026 - 22:59:28 | INFO | fastgen.callbacks.wandb:init_wandb:290 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable)
+[Apr 30, 2026 - 22:59:28 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully
+[Apr 30, 2026 - 22:59:28 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete
+[Apr 30, 2026 - 22:59:28 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer...
+[Apr 30, 2026 - 22:59:28 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully
+[Apr 30, 2026 - 22:59:28 | SUCCESS | __main__:main:33 ] Trainer initialized successfully
+[Apr 30, 2026 - 22:59:28 | INFO | fastgen.trainer:run:77 ] Starting training
+[Apr 30, 2026 - 22:59:28 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ...
+[Apr 30, 2026 - 22:59:28 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ...
+[Apr 30, 2026 - 22:59:30 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed
+[Apr 30, 2026 - 22:59:30 | INFO | fastgen.trainer:run:106 ] Wrapping model into ddp ..
+[Apr 30, 2026 - 22:59:30 | INFO | fastgen.trainer:run:108 ] DDP wrapping completed
+[Apr 30, 2026 - 22:59:30 | INFO | fastgen.networks.OmniAvatar.network_causal:_apply_unfreeze:1231 ] [CausalOmniAvatarWan] unfreeze: re-enabled requires_grad on 1.38M params in '_core.audio_proj'
+[Apr 30, 2026 - 22:59:30 | INFO | fastgen.networks.OmniAvatar.network_causal:_apply_unfreeze:1231 ] [CausalOmniAvatarWan] unfreeze: re-enabled requires_grad on 0.71M params in '_core.audio_cond_projs'
+[Apr 30, 2026 - 22:59:30 | INFO | fastgen.networks.OmniAvatar.network_causal:_apply_unfreeze:1231 ] [CausalOmniAvatarWan] unfreeze: re-enabled requires_grad on 0.40M params in '_core.patch_embedding'
+[Apr 30, 2026 - 22:59:30 | INFO | fastgen.networks.OmniAvatar.network_causal:apply_lora_freeze:1193 ] [CausalOmniAvatarWan] apply_lora_freeze: LoRA 174.98M trainable, base 1421.38M frozen, then unfreeze_modules re-enabled on top
+[Apr 30, 2026 - 22:59:30 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None
+[Apr 30, 2026 - 22:59:30 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 0.
+[Apr 30, 2026 - 22:59:30 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader...
+Traceback (most recent call last):
+  File "/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/train.py", line 46, in <module>
+    main(config)
+  File "/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/train.py", line 37, in main
+    fastgen_trainer.run(model)
+  File "/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/fastgen/trainer.py", line 166, in run
+    dataloader_train = instantiate(self.config.dataloader_train)
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/fastgen/utils/__init__.py", line 94, in instantiate
+    return cls(*args, **additional_kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/fastgen/datasets/omniavatar_dataloader.py", line 276, in __init__
+    self.dataset = OmniAvatarDataset(
+                   ^^^^^^^^^^^^^^^^^^
+  File "/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/fastgen/datasets/omniavatar_dataloader.py", line 135, in __init__
+    mask_img = Image.open(latentsync_mask_path)
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/PIL/Image.py", line 3505, in open
+    fp = builtins.open(filename, "rb")
+         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+FileNotFoundError: [Errno 2] No such file or directory: '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png'
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/train.py", line 46, in <module>
+[rank0]:     main(config)
+[rank0]:   File "/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/train.py", line 37, in main
+[rank0]:     fastgen_trainer.run(model)
+[rank0]:   File "/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/fastgen/trainer.py", line 166, in run
+[rank0]:     dataloader_train = instantiate(self.config.dataloader_train)
+[rank0]:                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/fastgen/utils/__init__.py", line 94, in instantiate
+[rank0]:     return cls(*args, **additional_kwargs)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/fastgen/datasets/omniavatar_dataloader.py", line 276, in __init__
+[rank0]:     self.dataset = OmniAvatarDataset(
+[rank0]:                    ^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/fastgen/datasets/omniavatar_dataloader.py", line 135, in __init__
+[rank0]:     mask_img = Image.open(latentsync_mask_path)
+[rank0]:                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.12/dist-packages/PIL/Image.py", line 3505, in open
+[rank0]:     fp = builtins.open(filename, "rb")
+[rank0]:          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]: FileNotFoundError: [Errno 2] No such file or directory: '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png'

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_225926-w9xbjand/files/requirements.txt

@@ -0,0 +1,417 @@
+DistVAE==0.0.0b5
+PySocks==1.7.1
+absl-py==2.4.0
+accelerate==1.13.0
+albucore==0.0.24
+albumentations==2.0.8
+antlr4-python3-runtime==4.9.3
+av==17.0.0
+blessed==1.33.0
+boto3==1.42.74
+botocore==1.42.74
+braceexpand==0.1.7
+cuda-bindings==13.2.0
+cuda-pathfinder==1.4.4
+cuda-toolkit==13.0.2
+decord==0.6.0
+diffusers==0.37.0
+dlib==20.0.0
+easydict==1.13
+flatbuffers==25.12.19
+ftfy==6.3.1
+gdown==5.2.1
+gitdb==4.0.12
+GitPython==3.1.46
+gpustat==1.1.1
+hf-xet==1.4.2
+huggingface_hub==1.7.2
+hydra-core==1.3.2
+ImageIO==2.37.3
+imageio-ffmpeg==0.6.0
+insightface==0.7.3
+jmespath==1.1.0
+kornia==0.8.2
+kornia_rs==0.1.10
+lmdb==2.2.0
+loguru==0.7.3
+mediapipe==0.10.18
+moviepy==2.2.1
+nvidia-cublas==13.1.0.3
+nvidia-cuda-cupti==13.0.85
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-runtime==13.0.96
+nvidia-cudnn-cu13==9.19.0.56
+nvidia-cufft==12.0.0.61
+nvidia-cufile==1.15.1.6
+nvidia-curand==10.4.0.35
+nvidia-cusolver==12.0.4.66
+nvidia-cusparse==12.6.3.3
+nvidia-cusparselt-cu13==0.8.0
+nvidia-nccl-cu13==2.28.9
+nvidia-nvjitlink==13.0.88
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvtx==13.0.85
+nvitop==1.6.2
+omegaconf==2.3.0
+onnxruntime==1.24.4
+onnxruntime-gpu==1.24.4
+open_clip_torch==3.3.0
+opencv-python-headless==4.11.0.86
+optimum-quanto==0.2.7
+peft==0.18.1
+prettytable==3.17.0
+proglog==0.1.12
+pyloudnorm==0.2.0
+python-dotenv==1.2.2
+python_speech_features==0.6
+pytorch-fid==0.3.0
+s3transfer==0.16.0
+scenedetect==0.6.7.1
+scikit-image==0.26.0
+sentencepiece==0.2.1
+sentry-sdk==2.55.0
+simsimd==6.5.16
+smmap==5.0.3
+sounddevice==0.5.5
+stringzilla==4.6.0
+tifffile==2026.3.3
+timm==1.0.26
+tokenizers==0.22.2
+transformers==5.3.0
+triton==3.2.0
+wandb==0.25.1
+wcwidth==0.6.0
+webdataset==1.0.2
+xfuser==0.4.5
+yunchang==0.6.4
+jupyterlab==4.5.0a1
+nvdlfw_inspect==0.1.0
+transformer_engine==2.3.0+5de3e14
+einops==0.8.1
+flash_attn==2.7.3
+torch_tensorrt==2.8.0a0
+apex==0.1
+nvidia-modelopt==0.27.1
+nvidia-modelopt-core==0.27.1
+nvidia-resiliency-ext==0.3.0
+polygraphy==0.49.20
+PuLP==3.1.1
+safetensors==0.5.3
+torchprofile==0.0.4
+tensorrt==10.10.0.31
+Cython==3.0.12
+aiohappyeyeballs==2.6.1
+aiohttp==3.11.16
+aiosignal==1.3.2
+asciitree==0.3.3
+cachetools==5.5.2
+charset-normalizer==3.4.1
+cloudpickle==3.1.1
+cuda-bindings==12.8.0
+cuda-python==12.8.0
+cudf==25.4.0
+cudf-polars==25.4.0
+cugraph==25.4.0
+cugraph-service-client==25.4.0
+cugraph-service-server==25.4.0
+cuml==25.4.0
+cupy-cuda12x==13.3.0
+cuvs==25.4.0
+dask==2025.2.0
+dask-cuda==25.4.0
+dask-cudf==25.4.0
+distributed==2025.2.0
+distributed-ucxx==0.43.0
+fasteners==0.19
+fastrlock==0.8.3
+frozenlist==1.5.0
+fsspec==2025.3.2
+grpcio==1.62.1
+joblib==1.4.2
+kvikio==25.4.0
+libcudf==25.4.0
+libcugraph==25.4.0
+libcuml==25.4.0
+libcuvs==25.4.0
+libkvikio==25.4.0
+libraft==25.4.0
+librmm==25.4.0
+librmm-cu12==25.4.0
+libucxx==0.43.0
+llvmlite==0.42.0
+locket==1.0.0
+multidict==6.4.3
+numba==0.59.1
+numba-cuda==0.4.0
+numcodecs==0.13.1
+nvidia-ml-py==12.570.86
+nvtx==0.2.11
+nx-cugraph==25.4.0
+packaging==23.2
+pandas==2.2.3
+partd==1.4.2
+ply==3.11
+polars==1.25.2
+propcache==0.3.1
+pyarrow==19.0.1
+pylibcudf==25.4.0
+pylibcugraph==25.4.0
+pylibraft==25.4.0
+pylibwholegraph==25.4.0
+pynvjitlink==0.3.0
+pynvml==12.0.0
+pytz==2023.4
+raft-dask==25.4.0
+rapids-dask-dependency==25.4.0a0
+rapids-logger==0.1.11
+rmm==25.4.0
+scipy==1.15.2
+tblib==3.1.0
+thriftpy2==0.5.2
+toolz==1.0.0
+treelite==4.4.1
+tzdata==2025.2
+ucx-py==0.43.0
+ucxx==0.43.0
+urllib3==2.0.7
+xgboost==2.1.3
+yarl==1.19.0
+zarr==2.18.7
+zict==3.0.0
+tabulate==0.9.0
+nvidia-cudnn-frontend==1.11.0
+onnx==1.17.0
+pybind11_global==2.13.6
+contourpy==1.3.2
+cycler==0.12.1
+fonttools==4.58.0
+kiwisolver==1.4.8
+matplotlib==3.10.3
+pycocotools==2.0+nv0.8.1
+pyparsing==3.2.3
+dm-tree==0.1.9
+gast==0.6.0
+nvidia-dali-cuda120==1.49.0
+nvidia-nvcomp-cu12==4.2.0.14
+nvidia-nvimgcodec-cu12==0.5.0.13
+nvidia-nvjpeg2k-cu12==0.8.1.40
+nvidia-nvjpeg-cu12==12.4.0.16
+nvidia-nvtiff-cu12==0.5.0.67
+pytorch-triton==3.3.0+git96316ce52.nvinternal
+lintrunner==0.12.7
+ninja==1.11.1.4
+optree==0.15.0
+setuptools==78.1.1
+types-dataclasses==0.6.6
+filelock==3.18.0
+mpmath==1.3.0
+networkx==3.4.2
+pillow==11.2.1
+sympy==1.14.0
+torch==2.8.0a0+5228986c39.nv25.5
+torchvision==0.22.0a0
+opencv==4.10.0
+black==25.1.0
+isort==6.0.1
+jupyterlab_tensorboard_pro==4.0.0
+jupytext==1.17.1
+mdit-py-plugins==0.4.2
+mypy_extensions==1.1.0
+pathspec==0.12.1
+Send2Trash==1.8.3
+absl-py==2.2.2
+anyio==4.9.0
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+asttokens==3.0.0
+async-lru==2.0.5
+babel==2.17.0
+beautifulsoup4==4.13.4
+bleach==6.2.0
+comm==0.2.2
+debugpy==1.8.14
+defusedxml==0.7.1
+executing==2.2.0
+fastjsonschema==2.21.1
+fqdn==1.5.1
+h11==0.16.0
+httpcore==1.0.9
+httpx==0.28.1
+ipykernel==6.29.5
+ipython==9.2.0
+ipython_pygments_lexers==1.1.1
+isoduration==20.11.0
+jedi==0.19.2
+json5==0.12.0
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2025.4.1
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+jupyter-events==0.12.0
+jupyter-lsp==2.2.5
+jupyter_server==2.16.0
+jupyter_server_terminals==0.5.3
+jupyterlab_code_formatter==3.0.2
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.27.3
+Markdown==3.8
+matplotlib-inline==0.1.7
+mistune==3.1.3
+nbclient==0.10.2
+nbconvert==7.16.6
+nbformat==5.10.4
+nest-asyncio==1.6.0
+notebook==7.4.2
+notebook_shim==0.2.4
+overrides==7.7.0
+pandocfilters==1.5.1
+parso==0.8.4
+pexpect==4.9.0
+prometheus_client==0.22.0
+prompt_toolkit==3.0.51
+psutil==7.0.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+python-dateutil==2.9.0.post0
+python-hostlist==2.2.1
+python-json-logger==3.3.0
+pyzmq==26.4.0
+referencing==0.36.2
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rpds-py==0.25.1
+sniffio==1.3.1
+soupsieve==2.7
+stack-data==0.6.3
+tensorboard==2.16.2
+tensorboard-data-server==0.7.2
+terminado==0.18.1
+tinycss2==1.4.0
+tornado==6.5
+traitlets==5.14.3
+types-python-dateutil==2.9.0.20250516
+uri-template==1.3.0
+wcwidth==0.2.13
+webcolors==24.11.1
+webencodings==0.5.1
+websocket-client==1.8.0
+Werkzeug==3.1.3
+MarkupSafe==3.0.2
+PyYAML==6.0.2
+annotated-types==0.7.0
+astunparse==1.6.3
+attrs==25.3.0
+audioread==3.0.1
+blis==0.7.11
+catalogue==2.0.10
+certifi==2025.4.26
+cffi==1.17.1
+click==8.1.8
+cloudpathlib==0.21.1
+confection==0.1.5
+cymem==2.0.11
+decorator==5.2.1
+execnet==2.1.1
+expecttest==0.3.0
+hypothesis==6.130.8
+idna==3.10
+iniconfig==2.1.0
+intel-openmp==2021.4.0
+Jinja2==3.1.6
+langcodes==3.5.0
+language_data==1.3.0
+lazy_loader==0.4
+librosa==0.11.0
+marisa-trie==1.2.1
+markdown-it-py==3.0.0
+mdurl==0.1.2
+mkl==2021.1.1
+mkl-devel==2021.1.1
+mkl-include==2021.1.1
+mock==5.2.0
+msgpack==1.1.0
+murmurhash==1.0.12
+numpy==1.26.4
+platformdirs==4.3.8
+pluggy==1.6.0
+pooch==1.8.2
+preshed==3.0.9
+protobuf==4.24.4
+pybind11==2.13.6
+pycparser==2.22
+pydantic==2.11.4
+pydantic_core==2.33.2
+Pygments==2.19.1
+pytest==8.1.1
+pytest-flakefinder==1.1.0
+pytest-rerunfailures==15.1
+pytest-shard==0.1.2
+pytest-xdist==3.6.1
+regex==2024.11.6
+requests==2.32.3
+rich==14.0.0
+scikit-learn==1.6.1
+shellingham==1.5.4
+six==1.16.0
+smart-open==7.1.0
+sortedcontainers==2.4.0
+soundfile==0.13.1
+soxr==0.5.0.post1
+spacy==3.7.5
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+srsly==2.5.1
+tbb==2021.13.1
+thinc==8.2.5
+threadpoolctl==3.6.0
+tqdm==4.67.1
+typer==0.15.4
+typing_extensions==4.13.2
+typing-inspection==0.4.1
+wasabi==1.1.3
+weasel==0.4.1
+wheel==0.45.1
+wrapt==1.17.2
+xdoctest==1.0.2
+cmake==3.31.6
+pip==25.1.1
+nvfuser==0.2.27a0+9bf5aca
+lightning-thunder==0.2.3.dev0
+dill==0.4.0
+opt_einsum==3.4.0
+lightning-utilities==0.14.3
+looseversion==1.3.0
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+inflect==7.3.1
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+more-itertools==10.3.0
+packaging==24.2
+platformdirs==4.2.2
+tomli==2.0.1
+typeguard==4.3.0
+typing_extensions==4.12.2
+wheel==0.45.1
+zipp==3.19.2
+PyGObject==3.48.2
+PyJWT==2.7.0
+blinker==1.7.0
+cryptography==41.0.7
+cryptography==41.0.7
+dbus-python==1.3.2
+distro==1.9.0
+httplib2==0.20.4
+launchpadlib==1.11.0
+lazr.restfulclient==0.14.6
+lazr.uri==1.0.6
+oauthlib==3.2.2
+pyparsing==3.1.1
+six==1.16.0
+ssh-import-id==5.11
+wadllib==1.3.6

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_225926-w9xbjand/files/wandb-metadata.json

@@ -0,0 +1,102 @@
+{
+  "os": "Linux-5.15.0-142-generic-x86_64-with-glibc2.39",
+  "python": "CPython 3.12.3",
+  "startedAt": "2026-04-30T13:59:26.861268Z",
+  "args": [
+    "--config=fastgen/configs/experiments/OmniAvatar/config_df_shift_5_lora_t769.py",
+    "-",
+    "dataloader_train.batch_size=8",
+    "trainer.ddp=True",
+    "trainer.max_iter=5000",
+    "trainer.save_ckpt_iter=500",
+    "trainer.resume=False",
+    "log_config.group=omniavatar_df_audiofix",
+    "log_config.name=df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter",
+    "log_config.project=OmniAvatar-FastGen",
+    "log_config.wandb_entity=paulhcho"
+  ],
+  "program": "/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/train.py",
+  "codePath": "train.py",
+  "codePathLocal": "train.py",
+  "git": {
+    "remote": "https://github.com/paulcho98/FastGen.git",
+    "commit": "f5897815a4bdcc8f11212b24f367a6839a5409ca"
+  },
+  "email": "paul.hyunbin@gmail.com",
+  "root": "/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/output/DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter",
+  "host": "instance-35250",
+  "executable": "/usr/bin/python",
+  "cpu_count": 64,
+  "cpu_count_logical": 128,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1599530123264",
+      "used": "753074597888"
+    }
+  },
+  "memory": {
+    "total": "1081693446144"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-c1619da3-e2c1-15b0-cb55-50e24b1f8e99"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-29f274f2-7065-4653-dcd1-055518ba099c"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-47a2dd51-a5dc-1d24-4713-ea97b7049d18"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-ad40d9cb-0070-8a6c-f0a3-d8d40d8233a7"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-97413b12-6c45-b8cd-a06a-886dd3bc94d6"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-060cf5c3-ba18-6ff1-9d4d-9e9c71bf6c67"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-6562d339-d2bb-95e2-3881-199a64c50ba7"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-a6834410-f241-7270-f5e3-129ecdc2e8f3"
+    }
+  ],
+  "cudaVersion": "12.9",
+  "writerId": "7agpz9rvil27ulbaooy9tu3j71z6whib"
+}

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_225926-w9xbjand/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":59},"_runtime":59}

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_225926-w9xbjand/logs/debug-internal.log

@@ -0,0 +1,24 @@
+{"time":"2026-04-30T22:59:27.121240873+09:00","level":"INFO","msg":"wandb-core"}
+{"time":"2026-04-30T22:59:27.127725168+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.1"}
+{"time":"2026-04-30T22:59:27.628090861+09:00","level":"INFO","msg":"stream: created new stream","id":"w9xbjand"}
+{"time":"2026-04-30T22:59:27.628168306+09:00","level":"INFO","msg":"handler: started"}
+{"time":"2026-04-30T22:59:27.629745961+09:00","level":"INFO","msg":"stream: started"}
+{"time":"2026-04-30T22:59:27.629773572+09:00","level":"INFO","msg":"sender: started"}
+{"time":"2026-04-30T22:59:27.629761079+09:00","level":"INFO","msg":"writer: started","stream_id":"w9xbjand"}
+{"time":"2026-04-30T22:59:28.861499272+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-04-30T22:59:29.336054055+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T22:59:43.861655852+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":19,"uploaded_len":2}
+{"time":"2026-04-30T22:59:44.215635379+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T22:59:58.861320253+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-04-30T22:59:59.140835619+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:00:13.861239149+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
+{"time":"2026-04-30T23:00:14.149003233+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:00:28.273108243+09:00","level":"INFO","msg":"stream: closing"}
+{"time":"2026-04-30T23:00:28.8610536+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"console_offset":19,"console_lines":42,"uploaded_len":2}
+{"time":"2026-04-30T23:00:28.8835215+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-04-30T23:00:29.203002489+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:00:29.203184098+09:00","level":"INFO","msg":"filestream: sending request","total_files":0,"uploaded_len":1,"complete":true,"exit_code":1}
+{"time":"2026-04-30T23:00:29.48012593+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:00:29.480401805+09:00","level":"INFO","msg":"handler: closed"}
+{"time":"2026-04-30T23:00:29.482558051+09:00","level":"INFO","msg":"sender: closed"}
+{"time":"2026-04-30T23:00:29.482571335+09:00","level":"INFO","msg":"stream: closed"}

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_225926-w9xbjand/logs/debug.log

@@ -0,0 +1,21 @@
+2026-04-30 22:59:26,882 INFO    MainThread:2764132 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
+2026-04-30 22:59:26,882 INFO    MainThread:2764132 [wandb_setup.py:_flush():81] Configure stats pid to 2764132
+2026-04-30 22:59:26,882 INFO    MainThread:2764132 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-30 22:59:26,882 INFO    MainThread:2764132 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/output/DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_225926-w9xbjand/logs/debug.log
+2026-04-30 22:59:26,882 INFO    MainThread:2764132 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/output/DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_225926-w9xbjand/logs/debug-internal.log
+2026-04-30 22:59:26,882 INFO    MainThread:2764132 [wandb_init.py:init():844] calling init triggers
+2026-04-30 22:59:26,882 INFO    MainThread:2764132 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_df_audiofix', 'name': 'df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/output/DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': "<class 'fastgen.callbacks.grad_clip.GradClipCallback'>"}, 'gpu_stats': {'every_n': '100', '_target_': "<class 'fastgen.callbacks.gpu_stats.GPUStatsCallback'>"}, 'train_profiler': {'every_n': '100', '_target_': "<class 'fastgen.callbacks.train_profiler.TrainProfilerCallback'>"}, 'param_count': {'_target_': "<class 'fastgen.callbacks.param_count.ParamCountCallback'>"}, 'wandb': {'sample_logging_iter': '500', 'fps': '25', '_target_': "<class 'fastgen.callbacks.wandb.WandbCallback'>"}}, 'save_ckpt_iter': '500', 'validation_iter': '500', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '5000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'True', 'fsdp': 'False', 'tf32_enabled': 'True', 'grad_accum_rounds': '1', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '4', 'neg_text_emb_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/text_emb/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': "<class 'fastgen.datasets.omniavatar_dataloader.OmniAvatarDataLoader'>"}, 'dataloader_val': {'data_list_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/text_emb/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': '<function create_omniavatar_dataloader at 0x7f7b29e018a0>'}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/OmniAvatar-Train/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data_checkpoints/1.3B-step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'use_dynamic_rope': 'False', 'stochastic_attn_configs': [{'local_attn_size': '7', 'sink_size': '1', 'weight': '0.2'}, {'local_attn_size': '10', 'sink_size': '1', 'weight': '0.2'}, {'local_attn_size': '13', 'sink_size': '1', 'weight': '0.2'}, {'local_attn_size': '9', 'sink_size': '3', 'weight': '0.2'}, {'local_attn_size': '12', 'sink_size': '3', 'weight': '0.2'}], '_target_': "<class 'fastgen.networks.OmniAvatar.network_causal.CausalOmniAvatarWan'>", 'merge_lora': 'False', 'unfreeze_modules': ['_core.audio_proj', '_core.audio_cond_projs', '_core.patch_embedding'], 'lora_rank': '128', 'lora_alpha': '64'}, 'teacher': None, 'fake_score_net': None, 'guidance_scale': None, 'timestep_cfg': {'enabled': 'False', 't_lo': '0.0', 't_hi': '1.0'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '1e-05', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7f7b2c301940>'}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'True', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'context_noise': '0.0', 'vae_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/OmniAvatar-Train/pretrained_models/Wan2.1-T2V-1.3B/Wan2.1_VAE.pth'}, 'model_class': {'config': None, '_target_': "<class 'fastgen.methods.omniavatar_diffusion_forcing.OmniAvatarDiffusionForcingModel'>"}, '_wandb': {}}
+2026-04-30 22:59:26,882 INFO    MainThread:2764132 [wandb_init.py:init():892] starting backend
+2026-04-30 22:59:27,107 INFO    MainThread:2764132 [wandb_init.py:init():895] sending inform_init request
+2026-04-30 22:59:27,118 INFO    MainThread:2764132 [wandb_init.py:init():903] backend started and connected
+2026-04-30 22:59:27,121 INFO    MainThread:2764132 [wandb_init.py:init():973] updated telemetry
+2026-04-30 22:59:27,162 INFO    MainThread:2764132 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-04-30 22:59:28,542 INFO    MainThread:2764132 [wandb_init.py:init():1042] starting run threads in backend
+2026-04-30 22:59:28,855 INFO    MainThread:2764132 [wandb_run.py:_console_start():2524] atexit reg
+2026-04-30 22:59:28,855 INFO    MainThread:2764132 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-04-30 22:59:28,855 INFO    MainThread:2764132 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-04-30 22:59:28,855 INFO    MainThread:2764132 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-04-30 22:59:28,859 INFO    MainThread:2764132 [wandb_init.py:init():1082] run started, returning control to user process
+2026-04-30 23:00:28,271 INFO    wandb-AsyncioManager-main:2764132 [service_client.py:_forward_responses():134] Reached EOF.
+2026-04-30 23:00:28,272 INFO    wandb-AsyncioManager-main:2764132 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_122daebb61ef6826b8de.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:122daebb61ef6826b8dee51f1938139a654d163927b7a901fab656eca1d70d4c
+size 173173

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_479372da7ed9e768cc22.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:479372da7ed9e768cc22ca68b676f84dc5bc2d17ad4a1b69d9e0603d07062ff0
+size 190184

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_509f08fac994ab8cebb6.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:509f08fac994ab8cebb6e804de594b7eb9f9167ea4c76921bb5f72028356b14f
+size 165124

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_5d1790cb2d9c8554b75b.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d1790cb2d9c8554b75bd606662be361f36a874249939a0be5a170c7592b69cf
+size 249191

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_81b3c24ab2930c8fd6c1.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81b3c24ab2930c8fd6c11e679d51bcdeadef1e799803868d89e8723f2a66ae92
+size 218665

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_8495e04002c0da67769b.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8495e04002c0da67769bc1c07b49ad88f9f86bb3f1569495ee69e5d26c2b032b
+size 223672

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_88d3eab6ac986b51e272.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88d3eab6ac986b51e27203bcd7c70a572b2166615c1abb27f2cc57fc335c0d04
+size 181968

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_8bcc788bab53b0c99f6b.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bcc788bab53b0c99f6b149c63c3b5c459461b6817a0082b55d46850c8937959
+size 171165

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_962d9fe735e704d64274.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:962d9fe735e704d642747fbde8da16feffd849351360204fefacb55f79107665
+size 227679

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/media/videos/val_gt/videos_0_b38481beb241c6eca7c1.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b38481beb241c6eca7c12d0e9e51785568d594ad93e71cc2d900d4a9e0a52a8e
+size 165222

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/output.log

@@ -0,0 +1,74 @@
+[Apr 30, 2026 - 23:05:49 | INFO | fastgen.callbacks.wandb:init_wandb:290 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable)
+[Apr 30, 2026 - 23:05:49 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully
+[Apr 30, 2026 - 23:05:49 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete
+[Apr 30, 2026 - 23:05:49 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer...
+[Apr 30, 2026 - 23:05:49 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully
+[Apr 30, 2026 - 23:05:49 | SUCCESS | __main__:main:33 ] Trainer initialized successfully
+[Apr 30, 2026 - 23:05:49 | INFO | fastgen.trainer:run:77 ] Starting training
+[Apr 30, 2026 - 23:05:49 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ...
+[Apr 30, 2026 - 23:05:49 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ...
+[Apr 30, 2026 - 23:05:50 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed
+[Apr 30, 2026 - 23:05:50 | INFO | fastgen.trainer:run:106 ] Wrapping model into ddp ..
+[Apr 30, 2026 - 23:05:51 | INFO | fastgen.trainer:run:108 ] DDP wrapping completed
+[Apr 30, 2026 - 23:05:51 | INFO | fastgen.networks.OmniAvatar.network_causal:_apply_unfreeze:1231 ] [CausalOmniAvatarWan] unfreeze: re-enabled requires_grad on 1.38M params in '_core.audio_proj'
+[Apr 30, 2026 - 23:05:51 | INFO | fastgen.networks.OmniAvatar.network_causal:_apply_unfreeze:1231 ] [CausalOmniAvatarWan] unfreeze: re-enabled requires_grad on 0.71M params in '_core.audio_cond_projs'
+[Apr 30, 2026 - 23:05:51 | INFO | fastgen.networks.OmniAvatar.network_causal:_apply_unfreeze:1231 ] [CausalOmniAvatarWan] unfreeze: re-enabled requires_grad on 0.40M params in '_core.patch_embedding'
+[Apr 30, 2026 - 23:05:51 | INFO | fastgen.networks.OmniAvatar.network_causal:apply_lora_freeze:1193 ] [CausalOmniAvatarWan] apply_lora_freeze: LoRA 174.98M trainable, base 1421.38M frozen, then unfreeze_modules re-enabled on top
+[Apr 30, 2026 - 23:05:51 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None
+[Apr 30, 2026 - 23:05:51 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 0.
+[Apr 30, 2026 - 23:05:51 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader...
+[Apr 30, 2026 - 23:06:47 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:438 ] Uploading GT validation videos to wandb...
+/usr/local/lib/python3.12/dist-packages/torchvision/io/_video_deprecation_warning.py:5: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec
+  warnings.warn(
+wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes.
+/usr/local/lib/python3.12/dist-packages/torchvision/io/_video_deprecation_warning.py:5: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec
+  warnings.warn(
+wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes.
+/usr/local/lib/python3.12/dist-packages/torchvision/io/_video_deprecation_warning.py:5: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec
+  warnings.warn(
+wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes.
+/usr/local/lib/python3.12/dist-packages/torchvision/io/_video_deprecation_warning.py:5: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec
+  warnings.warn(
+wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes.
+/usr/local/lib/python3.12/dist-packages/torchvision/io/_video_deprecation_warning.py:5: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec
+  warnings.warn(
+wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes.
+/usr/local/lib/python3.12/dist-packages/torchvision/io/_video_deprecation_warning.py:5: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec
+  warnings.warn(
+wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes.
+/usr/local/lib/python3.12/dist-packages/torchvision/io/_video_deprecation_warning.py:5: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec
+  warnings.warn(
+wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes.
+/usr/local/lib/python3.12/dist-packages/torchvision/io/_video_deprecation_warning.py:5: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec
+  warnings.warn(
+wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes.
+/usr/local/lib/python3.12/dist-packages/torchvision/io/_video_deprecation_warning.py:5: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec
+  warnings.warn(
+wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes.
+/usr/local/lib/python3.12/dist-packages/torchvision/io/_video_deprecation_warning.py:5: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec
+  warnings.warn(
+wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes.
+[Apr 30, 2026 - 23:07:34 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:463 ] Uploaded 10 GT validation videos to wandb
+[Apr 30, 2026 - 23:07:35 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1
+[Apr 30, 2026 - 23:07:35 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1
+[Apr 30, 2026 - 23:07:35 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarDiffusionForcingModel) has 177.47 M trainable and 1596.36 M total params (logical).
+[Apr 30, 2026 - 23:07:35 | INFO | fastgen.callbacks.param_count:on_train_begin:96 ] model (OmniAvatarDiffusionForcingModel) is NOT sharded (local == logical params).
+[Apr 30, 2026 - 23:07:35 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 177.47 M trainable and 1596.36 M total params (logical).
+[Apr 30, 2026 - 23:07:35 | INFO | fastgen.callbacks.param_count:on_train_begin:96 ] net (CausalOmniAvatarWan) is NOT sharded (local == logical params).
+[Apr 30, 2026 - 23:07:35 | INFO | fastgen.trainer:run:174 ] iter_start: 0
+[rank0]:E0430 23:07:57.183000 2767876 torch/_inductor/select_algorithm.py:2133] [0/0] Runtime error during autotuning:
+[rank0]:E0430 23:07:57.183000 2767876 torch/_inductor/select_algorithm.py:2133] [0/0] No valid triton configs. OutOfResources: out of resource: shared memory, Required: 167968, Hardware limit: 166912. Reducing block sizes or `num_stages` may help..
+[rank0]:E0430 23:07:57.183000 2767876 torch/_inductor/select_algorithm.py:2133] [0/0] Ignoring this choice.
+AUTOTUNE flex_attention(8x12x21504x128, 8x12x21504x128, 8x12x21504x128, 8x12x21504, 1x1x168, 1x1x168x168, 1x1x168, 1x1x168x168, 21504, 21504)
+strides: [33030144, 128, 1536, 1], [33030144, 128, 1536, 1], [33030144, 128, 1536, 1], [258048, 21504, 1], [168, 168, 1], [28224, 28224, 168, 1], [168, 168, 1], [28224, 28224, 168, 1], [1], [1]
+dtypes: torch.bfloat16, torch.bfloat16, torch.bfloat16, torch.float32, torch.int32, torch.int32, torch.int32, torch.int32, torch.int64, torch.int64
+  triton_flex_attention_0 313.7815 ms 100.0% BLOCKS_ARE_CONTIGUOUS=False, BLOCK_M=128, BLOCK_N=64, FLOAT32_PRECISION="'tf32'", GQA_SHARED_HEADS=1, HAS_FULL_BLOCKS=True, IS_DIVISIBLE=True, OUTPUT_LOGSUMEXP=True, PRESCALE_QK=False, QK_HEAD_DIM=128, QK_HEAD_DIM_ROUNDED=128, ROWS_GUARANTEED_SAFE=False, SAFE_HEAD_DIM=True, SM_SCALE=0.08838834764831843, SPARSE_KV_BLOCK_SIZE=128, SPARSE_Q_BLOCK_SIZE=128, V_HEAD_DIM=128, V_HEAD_DIM_ROUNDED=128, WRITE_DQ=True, num_stages=3, num_warps=8
+  triton_flex_attention_3 317.5469 ms 98.8% BLOCKS_ARE_CONTIGUOUS=False, BLOCK_M=128, BLOCK_N=128, FLOAT32_PRECISION="'tf32'", GQA_SHARED_HEADS=1, HAS_FULL_BLOCKS=True, IS_DIVISIBLE=True, OUTPUT_LOGSUMEXP=True, PRESCALE_QK=False, QK_HEAD_DIM=128, QK_HEAD_DIM_ROUNDED=128, ROWS_GUARANTEED_SAFE=False, SAFE_HEAD_DIM=True, SM_SCALE=0.08838834764831843, SPARSE_KV_BLOCK_SIZE=128, SPARSE_Q_BLOCK_SIZE=128, V_HEAD_DIM=128, V_HEAD_DIM_ROUNDED=128, WRITE_DQ=True, num_stages=2, num_warps=8
+  triton_flex_attention_4 423.0356 ms 74.2% BLOCKS_ARE_CONTIGUOUS=False, BLOCK_M=64, BLOCK_N=128, FLOAT32_PRECISION="'tf32'", GQA_SHARED_HEADS=1, HAS_FULL_BLOCKS=True, IS_DIVISIBLE=True, OUTPUT_LOGSUMEXP=True, PRESCALE_QK=False, QK_HEAD_DIM=128, QK_HEAD_DIM_ROUNDED=128, ROWS_GUARANTEED_SAFE=False, SAFE_HEAD_DIM=True, SM_SCALE=0.08838834764831843, SPARSE_KV_BLOCK_SIZE=128, SPARSE_Q_BLOCK_SIZE=128, V_HEAD_DIM=128, V_HEAD_DIM_ROUNDED=128, WRITE_DQ=True, num_stages=3, num_warps=4
+  triton_flex_attention_5 500.5695 ms 62.7% BLOCKS_ARE_CONTIGUOUS=False, BLOCK_M=64, BLOCK_N=64, FLOAT32_PRECISION="'tf32'", GQA_SHARED_HEADS=1, HAS_FULL_BLOCKS=True, IS_DIVISIBLE=True, OUTPUT_LOGSUMEXP=True, PRESCALE_QK=False, QK_HEAD_DIM=128, QK_HEAD_DIM_ROUNDED=128, ROWS_GUARANTEED_SAFE=False, SAFE_HEAD_DIM=True, SM_SCALE=0.08838834764831843, SPARSE_KV_BLOCK_SIZE=128, SPARSE_Q_BLOCK_SIZE=128, V_HEAD_DIM=128, V_HEAD_DIM_ROUNDED=128, WRITE_DQ=True, num_stages=3, num_warps=4
+  triton_flex_attention_1 713.1403 ms 44.0% BLOCKS_ARE_CONTIGUOUS=False, BLOCK_M=128, BLOCK_N=64, FLOAT32_PRECISION="'tf32'", GQA_SHARED_HEADS=1, HAS_FULL_BLOCKS=True, IS_DIVISIBLE=True, OUTPUT_LOGSUMEXP=True, PRESCALE_QK=False, QK_HEAD_DIM=128, QK_HEAD_DIM_ROUNDED=128, ROWS_GUARANTEED_SAFE=False, SAFE_HEAD_DIM=True, SM_SCALE=0.08838834764831843, SPARSE_KV_BLOCK_SIZE=128, SPARSE_Q_BLOCK_SIZE=128, V_HEAD_DIM=128, V_HEAD_DIM_ROUNDED=128, WRITE_DQ=True, num_stages=3, num_warps=4
+  triton_flex_attention_2 inf ms 0.0% BLOCKS_ARE_CONTIGUOUS=False, BLOCK_M=128, BLOCK_N=128, FLOAT32_PRECISION="'tf32'", GQA_SHARED_HEADS=1, HAS_FULL_BLOCKS=True, IS_DIVISIBLE=True, OUTPUT_LOGSUMEXP=True, PRESCALE_QK=False, QK_HEAD_DIM=128, QK_HEAD_DIM_ROUNDED=128, ROWS_GUARANTEED_SAFE=False, SAFE_HEAD_DIM=True, SM_SCALE=0.08838834764831843, SPARSE_KV_BLOCK_SIZE=128, SPARSE_Q_BLOCK_SIZE=128, V_HEAD_DIM=128, V_HEAD_DIM_ROUNDED=128, WRITE_DQ=True, num_stages=3, num_warps=4
+SingleProcess AUTOTUNE benchmarking takes 16.4067 seconds and 6.2248 seconds precompiling for 6 choices
+[rank0]:E0430 23:16:11.647000 2767876 torch/_inductor/select_algorithm.py:2133] [0/0] Runtime error during autotuning:
+[rank0]:E0430 23:16:11.647000 2767876 torch/_inductor/select_algorithm.py:2133] [0/0] No valid triton configs. OutOfResources: out of resource: shared memory, Required: 172032, Hardware limit: 166912. Reducing block sizes or `num_stages` may help..
+[rank0]:E0430 23:16:11.647000 2767876 torch/_inductor/select_algorithm.py:2133] [0/0] Ignoring this choice.

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/requirements.txt

@@ -0,0 +1,417 @@
+DistVAE==0.0.0b5
+PySocks==1.7.1
+absl-py==2.4.0
+accelerate==1.13.0
+albucore==0.0.24
+albumentations==2.0.8
+antlr4-python3-runtime==4.9.3
+av==17.0.0
+blessed==1.33.0
+boto3==1.42.74
+botocore==1.42.74
+braceexpand==0.1.7
+cuda-bindings==13.2.0
+cuda-pathfinder==1.4.4
+cuda-toolkit==13.0.2
+decord==0.6.0
+diffusers==0.37.0
+dlib==20.0.0
+easydict==1.13
+flatbuffers==25.12.19
+ftfy==6.3.1
+gdown==5.2.1
+gitdb==4.0.12
+GitPython==3.1.46
+gpustat==1.1.1
+hf-xet==1.4.2
+huggingface_hub==1.7.2
+hydra-core==1.3.2
+ImageIO==2.37.3
+imageio-ffmpeg==0.6.0
+insightface==0.7.3
+jmespath==1.1.0
+kornia==0.8.2
+kornia_rs==0.1.10
+lmdb==2.2.0
+loguru==0.7.3
+mediapipe==0.10.18
+moviepy==2.2.1
+nvidia-cublas==13.1.0.3
+nvidia-cuda-cupti==13.0.85
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-runtime==13.0.96
+nvidia-cudnn-cu13==9.19.0.56
+nvidia-cufft==12.0.0.61
+nvidia-cufile==1.15.1.6
+nvidia-curand==10.4.0.35
+nvidia-cusolver==12.0.4.66
+nvidia-cusparse==12.6.3.3
+nvidia-cusparselt-cu13==0.8.0
+nvidia-nccl-cu13==2.28.9
+nvidia-nvjitlink==13.0.88
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvtx==13.0.85
+nvitop==1.6.2
+omegaconf==2.3.0
+onnxruntime==1.24.4
+onnxruntime-gpu==1.24.4
+open_clip_torch==3.3.0
+opencv-python-headless==4.11.0.86
+optimum-quanto==0.2.7
+peft==0.18.1
+prettytable==3.17.0
+proglog==0.1.12
+pyloudnorm==0.2.0
+python-dotenv==1.2.2
+python_speech_features==0.6
+pytorch-fid==0.3.0
+s3transfer==0.16.0
+scenedetect==0.6.7.1
+scikit-image==0.26.0
+sentencepiece==0.2.1
+sentry-sdk==2.55.0
+simsimd==6.5.16
+smmap==5.0.3
+sounddevice==0.5.5
+stringzilla==4.6.0
+tifffile==2026.3.3
+timm==1.0.26
+tokenizers==0.22.2
+transformers==5.3.0
+triton==3.2.0
+wandb==0.25.1
+wcwidth==0.6.0
+webdataset==1.0.2
+xfuser==0.4.5
+yunchang==0.6.4
+jupyterlab==4.5.0a1
+nvdlfw_inspect==0.1.0
+transformer_engine==2.3.0+5de3e14
+einops==0.8.1
+flash_attn==2.7.3
+torch_tensorrt==2.8.0a0
+apex==0.1
+nvidia-modelopt==0.27.1
+nvidia-modelopt-core==0.27.1
+nvidia-resiliency-ext==0.3.0
+polygraphy==0.49.20
+PuLP==3.1.1
+safetensors==0.5.3
+torchprofile==0.0.4
+tensorrt==10.10.0.31
+Cython==3.0.12
+aiohappyeyeballs==2.6.1
+aiohttp==3.11.16
+aiosignal==1.3.2
+asciitree==0.3.3
+cachetools==5.5.2
+charset-normalizer==3.4.1
+cloudpickle==3.1.1
+cuda-bindings==12.8.0
+cuda-python==12.8.0
+cudf==25.4.0
+cudf-polars==25.4.0
+cugraph==25.4.0
+cugraph-service-client==25.4.0
+cugraph-service-server==25.4.0
+cuml==25.4.0
+cupy-cuda12x==13.3.0
+cuvs==25.4.0
+dask==2025.2.0
+dask-cuda==25.4.0
+dask-cudf==25.4.0
+distributed==2025.2.0
+distributed-ucxx==0.43.0
+fasteners==0.19
+fastrlock==0.8.3
+frozenlist==1.5.0
+fsspec==2025.3.2
+grpcio==1.62.1
+joblib==1.4.2
+kvikio==25.4.0
+libcudf==25.4.0
+libcugraph==25.4.0
+libcuml==25.4.0
+libcuvs==25.4.0
+libkvikio==25.4.0
+libraft==25.4.0
+librmm==25.4.0
+librmm-cu12==25.4.0
+libucxx==0.43.0
+llvmlite==0.42.0
+locket==1.0.0
+multidict==6.4.3
+numba==0.59.1
+numba-cuda==0.4.0
+numcodecs==0.13.1
+nvidia-ml-py==12.570.86
+nvtx==0.2.11
+nx-cugraph==25.4.0
+packaging==23.2
+pandas==2.2.3
+partd==1.4.2
+ply==3.11
+polars==1.25.2
+propcache==0.3.1
+pyarrow==19.0.1
+pylibcudf==25.4.0
+pylibcugraph==25.4.0
+pylibraft==25.4.0
+pylibwholegraph==25.4.0
+pynvjitlink==0.3.0
+pynvml==12.0.0
+pytz==2023.4
+raft-dask==25.4.0
+rapids-dask-dependency==25.4.0a0
+rapids-logger==0.1.11
+rmm==25.4.0
+scipy==1.15.2
+tblib==3.1.0
+thriftpy2==0.5.2
+toolz==1.0.0
+treelite==4.4.1
+tzdata==2025.2
+ucx-py==0.43.0
+ucxx==0.43.0
+urllib3==2.0.7
+xgboost==2.1.3
+yarl==1.19.0
+zarr==2.18.7
+zict==3.0.0
+tabulate==0.9.0
+nvidia-cudnn-frontend==1.11.0
+onnx==1.17.0
+pybind11_global==2.13.6
+contourpy==1.3.2
+cycler==0.12.1
+fonttools==4.58.0
+kiwisolver==1.4.8
+matplotlib==3.10.3
+pycocotools==2.0+nv0.8.1
+pyparsing==3.2.3
+dm-tree==0.1.9
+gast==0.6.0
+nvidia-dali-cuda120==1.49.0
+nvidia-nvcomp-cu12==4.2.0.14
+nvidia-nvimgcodec-cu12==0.5.0.13
+nvidia-nvjpeg2k-cu12==0.8.1.40
+nvidia-nvjpeg-cu12==12.4.0.16
+nvidia-nvtiff-cu12==0.5.0.67
+pytorch-triton==3.3.0+git96316ce52.nvinternal
+lintrunner==0.12.7
+ninja==1.11.1.4
+optree==0.15.0
+setuptools==78.1.1
+types-dataclasses==0.6.6
+filelock==3.18.0
+mpmath==1.3.0
+networkx==3.4.2
+pillow==11.2.1
+sympy==1.14.0
+torch==2.8.0a0+5228986c39.nv25.5
+torchvision==0.22.0a0
+opencv==4.10.0
+black==25.1.0
+isort==6.0.1
+jupyterlab_tensorboard_pro==4.0.0
+jupytext==1.17.1
+mdit-py-plugins==0.4.2
+mypy_extensions==1.1.0
+pathspec==0.12.1
+Send2Trash==1.8.3
+absl-py==2.2.2
+anyio==4.9.0
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+asttokens==3.0.0
+async-lru==2.0.5
+babel==2.17.0
+beautifulsoup4==4.13.4
+bleach==6.2.0
+comm==0.2.2
+debugpy==1.8.14
+defusedxml==0.7.1
+executing==2.2.0
+fastjsonschema==2.21.1
+fqdn==1.5.1
+h11==0.16.0
+httpcore==1.0.9
+httpx==0.28.1
+ipykernel==6.29.5
+ipython==9.2.0
+ipython_pygments_lexers==1.1.1
+isoduration==20.11.0
+jedi==0.19.2
+json5==0.12.0
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2025.4.1
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+jupyter-events==0.12.0
+jupyter-lsp==2.2.5
+jupyter_server==2.16.0
+jupyter_server_terminals==0.5.3
+jupyterlab_code_formatter==3.0.2
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.27.3
+Markdown==3.8
+matplotlib-inline==0.1.7
+mistune==3.1.3
+nbclient==0.10.2
+nbconvert==7.16.6
+nbformat==5.10.4
+nest-asyncio==1.6.0
+notebook==7.4.2
+notebook_shim==0.2.4
+overrides==7.7.0
+pandocfilters==1.5.1
+parso==0.8.4
+pexpect==4.9.0
+prometheus_client==0.22.0
+prompt_toolkit==3.0.51
+psutil==7.0.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+python-dateutil==2.9.0.post0
+python-hostlist==2.2.1
+python-json-logger==3.3.0
+pyzmq==26.4.0
+referencing==0.36.2
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rpds-py==0.25.1
+sniffio==1.3.1
+soupsieve==2.7
+stack-data==0.6.3
+tensorboard==2.16.2
+tensorboard-data-server==0.7.2
+terminado==0.18.1
+tinycss2==1.4.0
+tornado==6.5
+traitlets==5.14.3
+types-python-dateutil==2.9.0.20250516
+uri-template==1.3.0
+wcwidth==0.2.13
+webcolors==24.11.1
+webencodings==0.5.1
+websocket-client==1.8.0
+Werkzeug==3.1.3
+MarkupSafe==3.0.2
+PyYAML==6.0.2
+annotated-types==0.7.0
+astunparse==1.6.3
+attrs==25.3.0
+audioread==3.0.1
+blis==0.7.11
+catalogue==2.0.10
+certifi==2025.4.26
+cffi==1.17.1
+click==8.1.8
+cloudpathlib==0.21.1
+confection==0.1.5
+cymem==2.0.11
+decorator==5.2.1
+execnet==2.1.1
+expecttest==0.3.0
+hypothesis==6.130.8
+idna==3.10
+iniconfig==2.1.0
+intel-openmp==2021.4.0
+Jinja2==3.1.6
+langcodes==3.5.0
+language_data==1.3.0
+lazy_loader==0.4
+librosa==0.11.0
+marisa-trie==1.2.1
+markdown-it-py==3.0.0
+mdurl==0.1.2
+mkl==2021.1.1
+mkl-devel==2021.1.1
+mkl-include==2021.1.1
+mock==5.2.0
+msgpack==1.1.0
+murmurhash==1.0.12
+numpy==1.26.4
+platformdirs==4.3.8
+pluggy==1.6.0
+pooch==1.8.2
+preshed==3.0.9
+protobuf==4.24.4
+pybind11==2.13.6
+pycparser==2.22
+pydantic==2.11.4
+pydantic_core==2.33.2
+Pygments==2.19.1
+pytest==8.1.1
+pytest-flakefinder==1.1.0
+pytest-rerunfailures==15.1
+pytest-shard==0.1.2
+pytest-xdist==3.6.1
+regex==2024.11.6
+requests==2.32.3
+rich==14.0.0
+scikit-learn==1.6.1
+shellingham==1.5.4
+six==1.16.0
+smart-open==7.1.0
+sortedcontainers==2.4.0
+soundfile==0.13.1
+soxr==0.5.0.post1
+spacy==3.7.5
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+srsly==2.5.1
+tbb==2021.13.1
+thinc==8.2.5
+threadpoolctl==3.6.0
+tqdm==4.67.1
+typer==0.15.4
+typing_extensions==4.13.2
+typing-inspection==0.4.1
+wasabi==1.1.3
+weasel==0.4.1
+wheel==0.45.1
+wrapt==1.17.2
+xdoctest==1.0.2
+cmake==3.31.6
+pip==25.1.1
+nvfuser==0.2.27a0+9bf5aca
+lightning-thunder==0.2.3.dev0
+dill==0.4.0
+opt_einsum==3.4.0
+lightning-utilities==0.14.3
+looseversion==1.3.0
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+inflect==7.3.1
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+more-itertools==10.3.0
+packaging==24.2
+platformdirs==4.2.2
+tomli==2.0.1
+typeguard==4.3.0
+typing_extensions==4.12.2
+wheel==0.45.1
+zipp==3.19.2
+PyGObject==3.48.2
+PyJWT==2.7.0
+blinker==1.7.0
+cryptography==41.0.7
+cryptography==41.0.7
+dbus-python==1.3.2
+distro==1.9.0
+httplib2==0.20.4
+launchpadlib==1.11.0
+lazr.restfulclient==0.14.6
+lazr.uri==1.0.6
+oauthlib==3.2.2
+pyparsing==3.1.1
+six==1.16.0
+ssh-import-id==5.11
+wadllib==1.3.6

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/files/wandb-metadata.json

@@ -0,0 +1,104 @@
+{
+  "os": "Linux-5.15.0-142-generic-x86_64-with-glibc2.39",
+  "python": "CPython 3.12.3",
+  "startedAt": "2026-04-30T14:05:47.459514Z",
+  "args": [
+    "--config=fastgen/configs/experiments/OmniAvatar/config_df_shift_5_lora_t769.py",
+    "-",
+    "dataloader_train.batch_size=8",
+    "dataloader_train.latentsync_mask_path=/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/mask.png",
+    "dataloader_val.latentsync_mask_path=/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/mask.png",
+    "trainer.ddp=True",
+    "trainer.max_iter=5000",
+    "trainer.save_ckpt_iter=500",
+    "trainer.resume=False",
+    "log_config.group=omniavatar_df_audiofix",
+    "log_config.name=df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter",
+    "log_config.project=OmniAvatar-FastGen",
+    "log_config.wandb_entity=paulhcho"
+  ],
+  "program": "/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/train.py",
+  "codePath": "train.py",
+  "codePathLocal": "train.py",
+  "git": {
+    "remote": "https://github.com/paulcho98/FastGen.git",
+    "commit": "f5897815a4bdcc8f11212b24f367a6839a5409ca"
+  },
+  "email": "paul.hyunbin@gmail.com",
+  "root": "/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/output/DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter",
+  "host": "instance-35250",
+  "executable": "/usr/bin/python",
+  "cpu_count": 64,
+  "cpu_count_logical": 128,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1599530123264",
+      "used": "753074790400"
+    }
+  },
+  "memory": {
+    "total": "1081693446144"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-c1619da3-e2c1-15b0-cb55-50e24b1f8e99"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-29f274f2-7065-4653-dcd1-055518ba099c"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-47a2dd51-a5dc-1d24-4713-ea97b7049d18"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-ad40d9cb-0070-8a6c-f0a3-d8d40d8233a7"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-97413b12-6c45-b8cd-a06a-886dd3bc94d6"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-060cf5c3-ba18-6ff1-9d4d-9e9c71bf6c67"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-6562d339-d2bb-95e2-3881-199a64c50ba7"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-a6834410-f241-7270-f5e3-129ecdc2e8f3"
+    }
+  ],
+  "cudaVersion": "12.9",
+  "writerId": "d5096jhay1ocyc5oe5r6t4060xhmxcoy"
+}

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/logs/debug-internal.log

@@ -0,0 +1,145 @@
+{"time":"2026-04-30T23:05:47.717661886+09:00","level":"INFO","msg":"wandb-core"}
+{"time":"2026-04-30T23:05:47.719875838+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.1"}
+{"time":"2026-04-30T23:05:48.202102105+09:00","level":"INFO","msg":"stream: created new stream","id":"aogal2qe"}
+{"time":"2026-04-30T23:05:48.202176404+09:00","level":"INFO","msg":"handler: started"}
+{"time":"2026-04-30T23:05:48.203751244+09:00","level":"INFO","msg":"stream: started"}
+{"time":"2026-04-30T23:05:48.203784235+09:00","level":"INFO","msg":"sender: started"}
+{"time":"2026-04-30T23:05:48.203780548+09:00","level":"INFO","msg":"writer: started","stream_id":"aogal2qe"}
+{"time":"2026-04-30T23:05:49.289209394+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-04-30T23:05:49.628819533+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:06:04.28911266+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":19,"uploaded_len":2}
+{"time":"2026-04-30T23:06:04.68488743+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:06:19.28947862+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-04-30T23:06:19.593106092+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:06:34.289531575+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
+{"time":"2026-04-30T23:06:34.638885012+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:06:49.289408032+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":6,"events_lines":2,"console_offset":19,"console_lines":1}
+{"time":"2026-04-30T23:06:49.55441404+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:07:04.2893822+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
+{"time":"2026-04-30T23:07:04.547174503+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:07:19.289556072+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":10,"events_lines":2}
+{"time":"2026-04-30T23:07:19.542062737+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:07:34.289675411+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":12,"events_lines":2,"console_offset":20,"console_lines":29}
+{"time":"2026-04-30T23:07:34.590652485+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:07:49.289694092+09:00","level":"INFO","msg":"filestream: sending request","total_files":3,"events_offset":14,"events_lines":2,"console_offset":49,"console_lines":9,"uploaded_len":10}
+{"time":"2026-04-30T23:07:49.578907428+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:08:04.289321845+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":16,"events_lines":2,"console_offset":58,"console_lines":3}
+{"time":"2026-04-30T23:08:04.654848542+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:08:19.302053487+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":18,"events_lines":2,"console_offset":61,"console_lines":10}
+{"time":"2026-04-30T23:08:19.606327435+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:08:34.288993064+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":20,"events_lines":2}
+{"time":"2026-04-30T23:08:34.57863+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:08:49.289673912+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":22,"events_lines":2}
+{"time":"2026-04-30T23:08:49.597082041+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:09:04.289440794+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":24,"events_lines":2}
+{"time":"2026-04-30T23:09:04.62933334+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:09:19.289512374+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":26,"events_lines":2}
+{"time":"2026-04-30T23:09:19.579604059+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:09:34.288965379+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":28,"events_lines":2}
+{"time":"2026-04-30T23:09:34.594544765+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:09:49.288782859+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
+{"time":"2026-04-30T23:09:49.612205294+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:10:04.289135035+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":32,"events_lines":2}
+{"time":"2026-04-30T23:10:04.630551549+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:10:19.288780712+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":34,"events_lines":2}
+{"time":"2026-04-30T23:10:19.682644618+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:10:34.289481389+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":36,"events_lines":2}
+{"time":"2026-04-30T23:10:34.647039103+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:10:49.289265985+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":38,"events_lines":2}
+{"time":"2026-04-30T23:10:49.730743901+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:11:04.289172039+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":40,"events_lines":2}
+{"time":"2026-04-30T23:11:04.731077132+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:11:19.289448884+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":42,"events_lines":2}
+{"time":"2026-04-30T23:11:19.667453476+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:11:34.289592911+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":44,"events_lines":2}
+{"time":"2026-04-30T23:11:34.668128144+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:11:49.288778693+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":46,"events_lines":2}
+{"time":"2026-04-30T23:11:49.644035721+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:12:04.289349786+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":48,"events_lines":2}
+{"time":"2026-04-30T23:12:04.736082536+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:12:19.288945368+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":50,"events_lines":2}
+{"time":"2026-04-30T23:12:19.610329497+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:12:34.289571483+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":52,"events_lines":2}
+{"time":"2026-04-30T23:12:34.587850694+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:12:49.288764613+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":54,"events_lines":2}
+{"time":"2026-04-30T23:12:49.719622981+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:13:04.288637224+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":56,"events_lines":2}
+{"time":"2026-04-30T23:13:04.582812338+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:13:19.289301406+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":58,"events_lines":2}
+{"time":"2026-04-30T23:13:19.553204842+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:13:34.289451588+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":60,"events_lines":2}
+{"time":"2026-04-30T23:13:34.534556354+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:13:49.288622643+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":62,"events_lines":2}
+{"time":"2026-04-30T23:13:49.595316475+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:14:04.289144412+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":64,"events_lines":2}
+{"time":"2026-04-30T23:14:04.543244073+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:14:19.288823516+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":66,"events_lines":2}
+{"time":"2026-04-30T23:14:19.571833716+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:14:34.289509199+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":68,"events_lines":2}
+{"time":"2026-04-30T23:14:34.579585848+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:14:49.288824765+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":70,"events_lines":2}
+{"time":"2026-04-30T23:14:49.612924595+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:15:04.2892304+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":72,"events_lines":2}
+{"time":"2026-04-30T23:15:04.566368977+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:15:19.288719459+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":74,"events_lines":2}
+{"time":"2026-04-30T23:15:19.609636252+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:15:34.289310014+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":76,"events_lines":2}
+{"time":"2026-04-30T23:15:34.584427145+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:15:49.288956977+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":78,"events_lines":2}
+{"time":"2026-04-30T23:15:49.614829139+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:16:04.289218328+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":80,"events_lines":2}
+{"time":"2026-04-30T23:16:04.60993107+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:16:19.289171247+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":82,"events_lines":2,"console_offset":71,"console_lines":3}
+{"time":"2026-04-30T23:16:19.628814484+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:16:34.288676582+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":84,"events_lines":2}
+{"time":"2026-04-30T23:16:34.689081134+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:16:49.288933188+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":86,"events_lines":2}
+{"time":"2026-04-30T23:16:49.62914172+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:17:04.288607448+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":88,"events_lines":2}
+{"time":"2026-04-30T23:17:04.589594346+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:17:19.289321381+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":90,"events_lines":2}
+{"time":"2026-04-30T23:17:19.564264059+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:17:34.288994724+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":92,"events_lines":2}
+{"time":"2026-04-30T23:17:34.552863001+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:17:49.289503425+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":94,"events_lines":2}
+{"time":"2026-04-30T23:17:49.588243139+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:18:04.28878216+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":96,"events_lines":2}
+{"time":"2026-04-30T23:18:04.554107868+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:18:19.289179022+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":98,"events_lines":2}
+{"time":"2026-04-30T23:18:19.612210314+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:18:34.288875601+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":100,"events_lines":2}
+{"time":"2026-04-30T23:18:34.581426522+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:18:49.289382733+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":102,"events_lines":2}
+{"time":"2026-04-30T23:18:49.571645992+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:19:04.28863327+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":104,"events_lines":2}
+{"time":"2026-04-30T23:19:04.567218404+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:19:19.288592986+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":106,"events_lines":2}
+{"time":"2026-04-30T23:19:19.559133381+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:19:34.288571933+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":108,"events_lines":2}
+{"time":"2026-04-30T23:19:34.543783797+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:19:49.289279029+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":110,"events_lines":2}
+{"time":"2026-04-30T23:19:49.634365885+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:20:04.288850186+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":112,"events_lines":2}
+{"time":"2026-04-30T23:20:04.569464453+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:20:19.289155372+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":114,"events_lines":2}
+{"time":"2026-04-30T23:20:19.613994616+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:20:34.289328425+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":116,"events_lines":2}
+{"time":"2026-04-30T23:20:34.620516915+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:20:49.288670175+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":118,"events_lines":2}
+{"time":"2026-04-30T23:20:49.701752992+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:21:04.289448285+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":120,"events_lines":2}
+{"time":"2026-04-30T23:21:04.672086777+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:21:19.288957688+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":122,"events_lines":2}
+{"time":"2026-04-30T23:21:19.808028855+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:21:34.289496467+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":124,"events_lines":2}
+{"time":"2026-04-30T23:21:34.598270838+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:21:49.288582737+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":126,"events_lines":2}
+{"time":"2026-04-30T23:21:49.602869427+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:22:04.289027983+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":128,"events_lines":2}
+{"time":"2026-04-30T23:22:04.602488321+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:22:19.288843665+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":130,"events_lines":2}
+{"time":"2026-04-30T23:22:19.590625944+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:22:34.288973354+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":132,"events_lines":2}
+{"time":"2026-04-30T23:22:34.576595162+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-04-30T23:22:49.28941316+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":134,"events_lines":1}
+{"time":"2026-04-30T23:22:49.617243724+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/logs/debug.log

@@ -0,0 +1,19 @@
+2026-04-30 23:05:47,480 INFO    MainThread:2767876 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
+2026-04-30 23:05:47,480 INFO    MainThread:2767876 [wandb_setup.py:_flush():81] Configure stats pid to 2767876
+2026-04-30 23:05:47,480 INFO    MainThread:2767876 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-30 23:05:47,480 INFO    MainThread:2767876 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/output/DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/logs/debug.log
+2026-04-30 23:05:47,480 INFO    MainThread:2767876 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/output/DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/logs/debug-internal.log
+2026-04-30 23:05:47,480 INFO    MainThread:2767876 [wandb_init.py:init():844] calling init triggers
+2026-04-30 23:05:47,480 INFO    MainThread:2767876 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_df_audiofix', 'name': 'df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/FastGen/output/DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': "<class 'fastgen.callbacks.grad_clip.GradClipCallback'>"}, 'gpu_stats': {'every_n': '100', '_target_': "<class 'fastgen.callbacks.gpu_stats.GPUStatsCallback'>"}, 'train_profiler': {'every_n': '100', '_target_': "<class 'fastgen.callbacks.train_profiler.TrainProfilerCallback'>"}, 'param_count': {'_target_': "<class 'fastgen.callbacks.param_count.ParamCountCallback'>"}, 'wandb': {'sample_logging_iter': '500', 'fps': '25', '_target_': "<class 'fastgen.callbacks.wandb.WandbCallback'>"}}, 'save_ckpt_iter': '500', 'validation_iter': '500', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '5000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'True', 'fsdp': 'False', 'tf32_enabled': 'True', 'grad_accum_rounds': '1', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/mask.png', 'batch_size': '8', 'num_workers': '4', 'neg_text_emb_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/text_emb/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': "<class 'fastgen.datasets.omniavatar_dataloader.OmniAvatarDataLoader'>"}, 'dataloader_val': {'data_list_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data/text_emb/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': '<function create_omniavatar_dataloader at 0x7fa4931818a0>'}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/OmniAvatar-Train/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/data_checkpoints/1.3B-step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'use_dynamic_rope': 'False', 'stochastic_attn_configs': [{'local_attn_size': '7', 'sink_size': '1', 'weight': '0.2'}, {'local_attn_size': '10', 'sink_size': '1', 'weight': '0.2'}, {'local_attn_size': '13', 'sink_size': '1', 'weight': '0.2'}, {'local_attn_size': '9', 'sink_size': '3', 'weight': '0.2'}, {'local_attn_size': '12', 'sink_size': '3', 'weight': '0.2'}], '_target_': "<class 'fastgen.networks.OmniAvatar.network_causal.CausalOmniAvatarWan'>", 'merge_lora': 'False', 'unfreeze_modules': ['_core.audio_proj', '_core.audio_cond_projs', '_core.patch_embedding'], 'lora_rank': '128', 'lora_alpha': '64'}, 'teacher': None, 'fake_score_net': None, 'guidance_scale': None, 'timestep_cfg': {'enabled': 'False', 't_lo': '0.0', 't_hi': '1.0'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '1e-05', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': '<function get_optimizer at 0x7fa499799940>'}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': "<class 'fastgen.utils.lr_scheduler.LambdaLinearScheduler'>"}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'True', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'context_noise': '0.0', 'vae_path': '/data/karlo-research_715/workspace/kinemaar/paul/AR_diffusion/reference_FastGen_OmniAvatar/OmniAvatar-Train/pretrained_models/Wan2.1-T2V-1.3B/Wan2.1_VAE.pth'}, 'model_class': {'config': None, '_target_': "<class 'fastgen.methods.omniavatar_diffusion_forcing.OmniAvatarDiffusionForcingModel'>"}, '_wandb': {}}
+2026-04-30 23:05:47,480 INFO    MainThread:2767876 [wandb_init.py:init():892] starting backend
+2026-04-30 23:05:47,705 INFO    MainThread:2767876 [wandb_init.py:init():895] sending inform_init request
+2026-04-30 23:05:47,714 INFO    MainThread:2767876 [wandb_init.py:init():903] backend started and connected
+2026-04-30 23:05:47,716 INFO    MainThread:2767876 [wandb_init.py:init():973] updated telemetry
+2026-04-30 23:05:47,743 INFO    MainThread:2767876 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-04-30 23:05:48,972 INFO    MainThread:2767876 [wandb_init.py:init():1042] starting run threads in backend
+2026-04-30 23:05:49,282 INFO    MainThread:2767876 [wandb_run.py:_console_start():2524] atexit reg
+2026-04-30 23:05:49,283 INFO    MainThread:2767876 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-04-30 23:05:49,283 INFO    MainThread:2767876 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-04-30 23:05:49,283 INFO    MainThread:2767876 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-04-30 23:05:49,286 INFO    MainThread:2767876 [wandb_init.py:init():1082] run started, returning control to user process

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb/run-20260430_230547-aogal2qe/run-aogal2qe.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bd2a461c2164750645fcc7c85f423fce261f402d412a2e90a30baa22cd7d8ea
+size 262144

DF_LORA_T769_8GPU_BS8/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_lora_t769_8gpu_bs8_lr1e5_5000iter/wandb_id.txt

@@ -0,0 +1 @@
+aogal2qe