Update README.md

README.md

@@ -233,6 +233,148 @@ To run the script, follow these steps:
     </div>
 </div>
 
+### Run in Command line
+- animatediff_controlnet_sdxl_run_script.py
+```python
+import sys
+sys.path.insert(0, "diffusers-sdxl-controlnet/examples/community/")
+from animatediff_controlnet_sdxl import *
+
+import argparse
+from moviepy.editor import VideoFileClip, ImageSequenceClip
+import os
+import torch
+from diffusers.models import MotionAdapter
+from diffusers import DDIMScheduler, AutoPipelineForText2Image, ControlNetModel
+from diffusers.utils import export_to_gif
+from PIL import Image
+from controlnet_aux.processor import Processor
+
+# 初始化 MotionAdapter 和 ControlNetModel
+adapter = MotionAdapter.from_pretrained("a-r-r-o-w/animatediff-motion-adapter-sdxl-beta", torch_dtype=torch.float16)
+
+def initialize_pipeline(model_id):
+    scheduler = DDIMScheduler.from_pretrained(model_id, subfolder="scheduler", clip_sample=False, timestep_spacing="linspace", beta_schedule="linear", steps_offset=1)
+    controlnet = ControlNetModel.from_pretrained("thibaud/controlnet-openpose-sdxl-1.0", torch_dtype=torch.float16).to("cuda")
+
+    # 初始化 AnimateDiffSDXLControlnetPipeline
+    pipe = AnimateDiffSDXLControlnetPipeline.from_pretrained(
+        model_id,
+        controlnet=controlnet,
+        motion_adapter=adapter,
+        scheduler=scheduler,
+        torch_dtype=torch.float16,
+    ).to("cuda")
+    pipe.enable_vae_slicing()
+    pipe.enable_vae_tiling()
+    return pipe
+
+def split_video_into_frames(input_video_path, num_frames, temp_folder='temp_frames'):
+    """
+    将视频处理成指定帧数的视频，并保持原始的帧率。
+
+    :param input_video_path: 输入视频文件路径
+    :param num_frames: 目标帧数
+    :param temp_folder: 临时文件夹路径
+    """
+    clip = VideoFileClip(input_video_path)
+    original_duration = clip.duration
+    segment_duration = original_duration / num_frames
+
+    if not os.path.exists(temp_folder):
+        os.makedirs(temp_folder)
+
+    for i in range(num_frames):
+        frame_time = i * segment_duration
+        frame_path = os.path.join(temp_folder, f'frame_{i:04d}.png')
+        clip.save_frame(frame_path, t=frame_time)
+
+    frame_paths = [os.path.join(temp_folder, f'frame_{i:04d}.png') for i in range(num_frames)]
+    final_clip = ImageSequenceClip(frame_paths, fps=clip.fps)
+    final_clip.write_videofile("resampled_video.mp4", codec='libx264')
+
+    print(f"新的视频已保存到 resampled_video.mp4，包含 {num_frames} 个帧，并保持原始的帧率。")
+
+def generate_video_with_prompt(input_video_path, prompt, model_id, gif_output_path, num_frames=16, keep_imgs=False, temp_folder='temp_frames'):
+    """
+    生成带有文本提示的视频。
+
+    :param input_video_path: 输入视频文件路径
+    :param prompt: 文本提示
+    :param model_id: 模型ID
+    :param gif_output_path: GIF 输出文件路径
+    :param num_frames: 目标帧数
+    :param keep_imgs: 是否保留临时图片
+    :param temp_folder: 临时文件夹路径
+    """
+    split_video_into_frames(input_video_path, num_frames, temp_folder)
+
+    folder_path = temp_folder
+    frames = os.listdir(folder_path)
+    frames = list(filter(lambda x: x.endswith(".png"), frames))
+    frames.sort()
+    conditioning_frames = list(map(lambda x: Image.open(os.path.join(folder_path, x)).resize((1024, 1024)), frames))[:num_frames]
+
+    p2 = Processor("openpose")
+    cn2 = [p2(frame) for frame in conditioning_frames]
+
+    negative_prompt = "bad quality, worst quality, jpeg artifacts, ugly"
+    generator = torch.Generator(device="cpu").manual_seed(0)
+
+    pipe = initialize_pipeline(model_id)
+
+    output = pipe(
+        prompt=prompt,
+        negative_prompt=negative_prompt,
+        num_inference_steps=50,
+        guidance_scale=20,
+        controlnet_conditioning_scale=1.0,
+        width=512,
+        height=768,
+        num_frames=num_frames,
+        conditioning_frames=cn2,
+        generator=generator
+    )
+
+    frames = output.frames[0]
+    export_to_gif(frames, gif_output_path)
+
+    print(f"生成的 GIF 已保存到 {gif_output_path}")
+
+    if not keep_imgs:
+        # 删除临时文件夹
+        import shutil
+        shutil.rmtree(temp_folder)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="生成带有文本提示的视频")
+    parser.add_argument("input_video", help="输入视频文件路径")
+    parser.add_argument("prompt", help="文本提示")
+    parser.add_argument("model_id", help="模型ID")
+    parser.add_argument("gif_output_path", help="GIF 输出文件路径")
+    parser.add_argument("--num_frames", type=int, default=16, help="目标帧数")
+    parser.add_argument("--keep_imgs", action="store_true", help="是否保留临时图片")
+    parser.add_argument("--temp_folder", default='temp_frames', help="临时文件夹路径")
+
+    args = parser.parse_args()
+
+    generate_video_with_prompt(args.input_video, args.prompt, args.model_id, args.gif_output_path, args.num_frames, args.keep_imgs, args.temp_folder)
+
+```
+
+```bash
+python animatediff_controlnet_sdxl_run_script.py girl_beach.mp4 \
+ "solo,Xiangling\(genshin impact\),1girl,full body professional photograph of a stunning detailed, drink tea use chinese cup" \
+  "svjack/GenshinImpact_XL_Base" \
+   xiangling_tea_animation.gif --num_frames 16 --temp_folder temp_frames
+```
+- Pose: girl_beach.mp4
+<video controls autoplay src="https://cdn-uploads.huggingface.co/production/uploads/634dffc49b777beec3bc6448/pYx23VyLNkLk3YxAAqu5i.mp4"></video>
+- Output: xiangling_tea_animation.gif
+![image/gif](https://cdn-uploads.huggingface.co/production/uploads/634dffc49b777beec3bc6448/qUZOvGs5rzxN8zaZ4Xp3s.gif)
+- Upscaled:
+<video controls autoplay src="https://cdn-uploads.huggingface.co/production/uploads/634dffc49b777beec3bc6448/uwUDYOPiZbHuq5v6jWADr.mp4"></video>
+
 ## Conclusion
 
 This script demonstrates how to use the `diffusers-sdxl-controlnet` library to generate animated images with ControlNet and SDXL models. By following the steps outlined above, you can create and visualize your own animated sequences.