Upload alpha_work/configs/test_sd_1_vid_1_f_v2_new_bg_scale_v9_sm_v2_stage2_loadself_scale_test.yaml with huggingface_hub

alpha_work/configs/test_sd_1_vid_1_f_v2_new_bg_scale_v9_sm_v2_stage2_loadself_scale_test.yaml

@@ -0,0 +1,155 @@
+exp_root_dir: exp
+exp_name: layer_diffusion_sd_ft_3_vid_v4.2_new_bg_scale_v9_sm_v2_stage2_new_loadself_scale_test
+mode: test # test
+image_finetune: False
+
+# pretrained_model_path: /home/nfs/wyy/models/sd-image-variations-diffusers
+# pretrained_model_path: /mnt/workspace/workgroup/sihui.jsh/DisCo10.9/diffusers/RealisticVision-v2-1
+pretrained_model_path: diffusers_/stable-diffusion-v1-5
+# pretrained_unet_path: exp/ckpts/layer_diffusion_sd_ft_2/epoch_200.pth
+# pretrained_trans_lora: exp/ckpts/layer_diffusion_sd_ft_3_vid_img_v2.1/epoch_73.pth
+pretrained_trans_lora: exp/ckpts/layer_diffusion_sd_ft_3_vid_img_v2.2.1/epoch_40.pth
+# pretrained_trans_lora: ""
+# pretrained_vae_path: ../alpha_work/diffusers/LayerDiffusion
+pretrained_vae_path: exp/ckpts/layer_diffusion_2/epoch_6.pth
+data_root_dir: ../
+seed: 22
+
+load_state: True
+load_state_exp_name: layer_diffusion_sd_ft_3_vid_v4.2_new_bg_scale_v9_sm_v2_stage2_new_loadself_scale
+load_state_epoch: 18
+
+learning_rate: 0.00002
+decay: 0.001
+
+use_text_embedding: True
+text_encoder: clip
+
+save_state_epoch: 1
+num_epochs: 200
+batch_size: 1 #
+eval_batch_size: 1  #8
+eval_step: 3000
+eval_scheduler: ddim
+eval_image_num: 50
+
+image_encoder: dinov2
+refer_sdvae: True
+unet_trainable_module: up_spatial_lora # transformer, cross_attn, self_cross_attn, all
+ref_encoder_type: dinov2 # clip_all, clip_global, dinov2, clip_multi
+use_clip_proj: False
+cfg_zero_image_first: False
+
+num_inf_images_per_prompt: 1
+num_inference_steps: 25
+scale_factor: 0.18215
+guidance_scale: 8.5
+cfg_eta: 0.1
+controlnet_conditioning_scale: 1.0
+
+device: cuda
+mixed_precision: 'fp16' #'no','fp16','bf16 or 'fp8'
+enable_xformers_memory_efficient_attention: True
+gradient_checkpointing: True
+gradient_accumulate_steps: 2
+num_workers: 8
+
+unet_additional_kwargs:
+  use_inflated_groupnorm:     true
+  use_motion_module:          true
+  motion_module_resolutions:  [1,2,4,8]
+  motion_module_mid_block:    false
+  motion_module_type:         Vanilla
+
+  motion_module_kwargs:
+    num_attention_heads:                 8
+    num_transformer_block:               1
+    attention_block_types:               [ "Temporal_Self", "Temporal_Self" ]
+    temporal_position_encoding:          true
+    temporal_position_encoding_max_len:  32
+    temporal_attention_dim_div:          1
+    zero_initialize:                     true
+
+noise_scheduler_kwargs:
+  beta_start:    0.00085
+  beta_end:      0.012
+  beta_schedule: "linear"
+  steps_offset:  1
+  clip_sample:   False
+
+# train_data:
+# csv_train_path_list:        ["/mnt/workspace/workgroup/video_datasets/WebVid/meta/results_2M_train.csv"]
+# csv_test_path_list:        ["/mnt/workspace/workgroup/video_datasets/WebVid/meta/results_2M_train.csv"]
+# json_train_path_list:         ["../layer_diff_dataset/video/video_all.json"]
+# json_test_path_list:         ["../layer_diff_dataset/video/video_test.json"]
+# video_train_folder_list:    ["../layervid/MeViS/valid_u"]
+video_train_folder_list:    ['../data/DIS-TR']
+# video_train_folder_list:    ['../data/DIS-TR','../data/MULAN','../data/AM-2k/train',"../data/video_dataset/YoutubeVOS/train"]
+# video_test_folder_list:    ["../layervid/MeViS/valid_u"] 
+video_test_folder_list:    ["../data/video_dataset/YoutubeVOS/train"]
+# mask_train_folder_list:     ["/mnt/workspace/workgroup/yuanpeng/videoedit/webvid/Segment-and-Track-Anything/savemask/WebVid_short"]
+# mask_test_folder_list:     ["/mnt/workspace/workgroup/yuanpeng/videoedit/webvid/Segment-and-Track-Anything/savemask/WebVid_short"]
+train_json_name: "meta_v5.json"
+test_json_name: "meta_user_study.json"
+img_size: [256,256]
+sample_stride:  1
+sample_n_frames: 16
+num_frames: 16
+
+# adapter_lora_path: "../AnimateDiff/models/Motion_Module/v3_sd15_adapter.ckpt"
+adapter_lora_path: ""
+dreambooth_model_path:   "../AnimateDiff/models/DreamBooth_LoRA/realisticVisionV51_v51VAE.safetensors"
+# dreambooth_model_path:   ""
+lora_model_path: ""
+inference_config: "configs/inference/inference-v3.yaml"
+motion_module_path:    "../AnimateDiff/models/Motion_Module/v3_sd15_mm.ckpt"
+controlnet_config: "configs/inference/sparsectrl/latent_condition.yaml"
+controlnet_path:   "../AnimateDiff/models/SparseCtrl/v3_sd15_sparsectrl_rgb.ckpt"
+motion_module_lora_configs: []
+
+list_vae: ["encoder.mid_block.attentions.0.key.bias", "encoder.mid_block.attentions.0.key.weight", "encoder.mid_block.attentions.0.proj_attn.bias", "encoder.mid_block.attentions.0.proj_attn.weight", "encoder.mid_block.attentions.0.query.bias", "encoder.mid_block.attentions.0.query.weight", "encoder.mid_block.attentions.0.value.bias", "encoder.mid_block.attentions.0.value.weight", "decoder.mid_block.attentions.0.key.bias", "decoder.mid_block.attentions.0.key.weight", "decoder.mid_block.attentions.0.proj_attn.bias", "decoder.mid_block.attentions.0.proj_attn.weight", "decoder.mid_block.attentions.0.query.bias", "decoder.mid_block.attentions.0.query.weight", "decoder.mid_block.attentions.0.value.bias", "decoder.mid_block.attentions.0.value.weight"]
+frames: 3
+
+enable_lora: True
+motion_lora_rank: 4
+spatial_lora_rank: 64
+load_lora: True
+# load_lora_exp_name: layer_diffusion_sd_ft_3_vid_v4.2_lora_1
+load_lora_exp_name: layer_diffusion_sd_ft_3_vid_v4.2_new_bg_scale_v9_sm_v2.1
+load_lora_epoch: 5
+is_stage2: True
+
+module_mapping_sd15: {
+  0: 'down_blocks.0.attentions.0.transformer_blocks.0.attn1',
+  1: 'down_blocks.0.attentions.0.transformer_blocks.0.attn2',
+  2: 'down_blocks.0.attentions.1.transformer_blocks.0.attn1',
+  3: 'down_blocks.0.attentions.1.transformer_blocks.0.attn2',
+  4: 'down_blocks.1.attentions.0.transformer_blocks.0.attn1',
+  5: 'down_blocks.1.attentions.0.transformer_blocks.0.attn2',
+  6: 'down_blocks.1.attentions.1.transformer_blocks.0.attn1',
+  7: 'down_blocks.1.attentions.1.transformer_blocks.0.attn2',
+  8: 'down_blocks.2.attentions.0.transformer_blocks.0.attn1',
+  9: 'down_blocks.2.attentions.0.transformer_blocks.0.attn2',
+  10: 'down_blocks.2.attentions.1.transformer_blocks.0.attn1',
+  11: 'down_blocks.2.attentions.1.transformer_blocks.0.attn2',
+  12: 'up_blocks.1.attentions.0.transformer_blocks.0.attn1',
+  13: 'up_blocks.1.attentions.0.transformer_blocks.0.attn2',
+  14: 'up_blocks.1.attentions.1.transformer_blocks.0.attn1',
+  15: 'up_blocks.1.attentions.1.transformer_blocks.0.attn2',
+  16: 'up_blocks.1.attentions.2.transformer_blocks.0.attn1',
+  17: 'up_blocks.1.attentions.2.transformer_blocks.0.attn2',
+  18: 'up_blocks.2.attentions.0.transformer_blocks.0.attn1',
+  19: 'up_blocks.2.attentions.0.transformer_blocks.0.attn2',
+  20: 'up_blocks.2.attentions.1.transformer_blocks.0.attn1',
+  21: 'up_blocks.2.attentions.1.transformer_blocks.0.attn2',
+  22: 'up_blocks.2.attentions.2.transformer_blocks.0.attn1',
+  23: 'up_blocks.2.attentions.2.transformer_blocks.0.attn2',
+  24: 'up_blocks.3.attentions.0.transformer_blocks.0.attn1',
+  25: 'up_blocks.3.attentions.0.transformer_blocks.0.attn2',
+  26: 'up_blocks.3.attentions.1.transformer_blocks.0.attn1',
+  27: 'up_blocks.3.attentions.1.transformer_blocks.0.attn2',
+  28: 'up_blocks.3.attentions.2.transformer_blocks.0.attn1',
+  29: 'up_blocks.3.attentions.2.transformer_blocks.0.attn2',
+  30: 'mid_block.attentions.0.transformer_blocks.0.attn1',
+  31: 'mid_block.attentions.0.transformer_blocks.0.attn2'
+}