add readme

README.md

@@ -0,0 +1,110 @@
+# How to make this
+
+- pretrained model: [epiCRealism](https://civitai.com/models/25694?modelVersionId=134065) + [hyper CFG lora 12steps](https://huggingface.co/ByteDance/Hyper-SD/blob/main/Hyper-SD15-12steps-CFG-lora.safetensors)
+-> merge with lora weight 0.3
+- lora model: [AnimateLCM_sd15_t2v_lora.safetensors](https://huggingface.co/wangfuyun/AnimateLCM/blob/main/AnimateLCM_sd15_t2v_lora.safetensors)
+
+```python
+
+import torch
+
+from diffusers import StableDiffusionPipeline, DPMSolverMultistepScheduler
+
+from animatediff.models.unet import UNet3DConditionModel
+from animatediff.models.sparse_controlnet import SparseControlNetModel
+from animatediff.pipelines.pipeline_animation import AnimationPipeline
+from animatediff.utils.util import load_weights
+
+sdpipe = StableDiffusionPipeline.from_single_file(pretrained_model_path, use_safetensors=True, add_watermarker=False).to(dtype=torch.float16)
+sdpipe.load_lora_weights(lora_model_path)
+sdpipe.fuse_lora(lora_scale=0.3)
+
+text_encoder = sdpipe.text_encoder.cuda()
+vae          = sdpipe.vae.cuda()
+tokenizer    = sdpipe.tokenizer
+
+unet_additional_kwargs = params["unet_additional_kwargs"]
+controlnet_additional_kwargs = params["controlnet_additional_kwargs"]
+
+unet = UNet3DConditionModel.from_pretrained_2d(pretrained_model_path, subfolder="unet", unet_config=sdpipe.unet.config, unet_additional_kwargs=unet_additional_kwargs).cuda()
+unet.config.num_attention_heads = 8
+unet.config.projection_class_embeddings_input_dim = None
+unet.to(dtype=torch.float16)
+
+controlnet = SparseControlNetModel.from_unet(unet, controlnet_additional_kwargs=controlnet_additional_kwargs)
+controlnet_path = "models/motion_module/v3_sd15_sparsectrl_rgb.ckpt"
+
+print(f"loading controlnet checkpoint from {controlnet_path} ...")
+controlnet_state_dict = torch.load(controlnet_path, map_location="cpu")
+controlnet_state_dict = controlnet_state_dict["controlnet"] if "controlnet" in controlnet_state_dict else controlnet_state_dict
+controlnet_state_dict = {name: param for name, param in controlnet_state_dict.items() if "pos_encoder.pe" not in name}
+controlnet_state_dict.pop("animatediff_config", "")
+controlnet.load_state_dict(controlnet_state_dict)
+controlnet.to(dtype=torch.float16)
+controlnet.cuda()
+
+pipe = load_weights(
+    pipeline,
+    # motion module
+    motion_module_path         = "models/Motion_Module/v3_sd15_mm.ckpt", 
+    motion_module_lora_configs = [],
+    # domain adapter
+    adapter_lora_path          = "models/Motion_Module/v3_sd15_adapter.ckpt",
+    adapter_lora_scale         = 1.0,
+    # image layers
+    dreambooth_model_path      = pretrained_model_path, 
+    lora_model_path            = "",
+    lora_alpha                 = 0.8,
+).to("cuda")
+pipe.to(dtype=torch.float16)
+pipe.enable_vae_slicing()
+pipe.enable_model_cpu_offload()
+
+pipe.scheduler = DPMSolverMultistepScheduler(
+    beta_start = 0.00075,
+    beta_end = 0.0145,
+    beta_schedule = "linear",
+    use_karras_sigmas = True,
+)
+
+```
+
+# How to use this
+
+```python
+controlnet_additional_kwargs = params["controlnet_additional_kwargs"]
+
+pipe = AnimationPipeline.from_pretrained("models/animatediff_model")
+unet = pipe.unet
+vae = pipe.vae
+
+unet.config.num_attention_heads = 8
+unet.config.projection_class_embeddings_input_dim = None
+unet.to(dtype=torch.float16)
+
+controlnet = SparseControlNetModel.from_unet(unet, controlnet_additional_kwargs=controlnet_additional_kwargs)
+controlnet_path = "./models/motion_module/v3_sd15_sparsectrl_rgb.ckpt"
+
+print(f"loading controlnet checkpoint from {controlnet_path} ...")
+controlnet_state_dict = torch.load(controlnet_path, map_location="cpu")
+controlnet_state_dict = controlnet_state_dict["controlnet"] if "controlnet" in controlnet_state_dict else controlnet_state_dict
+controlnet_state_dict = {name: param for name, param in controlnet_state_dict.items() if "pos_encoder.pe" not in name}
+controlnet_state_dict.pop("animatediff_config", "")
+controlnet.load_state_dict(controlnet_state_dict)
+controlnet.to(dtype=torch.float16)
+controlnet.cuda()
+
+pipe.controlnet = controlnet
+
+without_xformers = False
+if is_xformers_available() and (not without_xformers):
+    unet.enable_xformers_memory_efficient_attention()
+    if controlnet is not None: 
+        print("\nenable_xformers_memory_efficient_attention\n")
+        controlnet.enable_xformers_memory_efficient_attention()
+
+pipe.to(dtype=torch.float16)
+pipe.enable_vae_slicing()
+pipe.enable_model_cpu_offload()
+pipe.to("cuda")
+```

scheduler/scheduler_config.json

@@ -8,7 +8,6 @@
   "dynamic_thresholding_ratio": 0.995,
   "euler_at_final": false,
   "final_sigmas_type": "zero",
-  "lambda_min_clipped": -Infinity,
   "lower_order_final": true,
   "num_train_timesteps": 1000,
   "prediction_type": "epsilon",