Upload 285 files

.gitattributes

@@ -17,3 +17,22 @@ ckpts/naturalspeech3_facodec/ns3_facodec_decoder.bin filter=lfs diff=lfs merge=l
 ckpts/naturalspeech3_facodec/ns3_facodec_encoder_v2.bin filter=lfs diff=lfs merge=lfs -text
 ckpts/naturalspeech3_facodec/ns3_facodec_encoder.bin filter=lfs diff=lfs merge=lfs -text
 ckpts/naturalspeech3_facodec/ns3_facodec_redecoder.bin filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/assets/examples/driving/d8.pkl filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/pretrained_weights/insightface/models/buffalo_l/2d106det.onnx filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/pretrained_weights/insightface/models/buffalo_l/det_10g.onnx filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/pretrained_weights/liveportrait_animals/base_models_v1.1/appearance_feature_extractor.pth filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/pretrained_weights/liveportrait_animals/base_models_v1.1/motion_extractor.pth filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/pretrained_weights/liveportrait_animals/base_models_v1.1/spade_generator.pth filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/pretrained_weights/liveportrait_animals/base_models_v1.1/warping_module.pth filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/pretrained_weights/liveportrait_animals/base_models/appearance_feature_extractor.pth filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/pretrained_weights/liveportrait_animals/base_models/motion_extractor.pth filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/pretrained_weights/liveportrait_animals/base_models/spade_generator.pth filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/pretrained_weights/liveportrait_animals/base_models/warping_module.pth filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/pretrained_weights/liveportrait_animals/retargeting_models/stitching_retargeting_module.pth filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/pretrained_weights/liveportrait_animals/xpose.pth filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/pretrained_weights/liveportrait/base_models/appearance_feature_extractor.pth filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/pretrained_weights/liveportrait/base_models/motion_extractor.pth filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/pretrained_weights/liveportrait/base_models/spade_generator.pth filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/pretrained_weights/liveportrait/base_models/warping_module.pth filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/pretrained_weights/liveportrait/landmark.onnx filter=lfs diff=lfs merge=lfs -text
+ckpts/LivePortrait/pretrained_weights/liveportrait/retargeting_models/stitching_retargeting_module.pth filter=lfs diff=lfs merge=lfs -text

ckpts/LivePortrait/.gitignore

@@ -0,0 +1,34 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+**/__pycache__/
+*.py[cod]
+**/*.py[cod]
+*$py.class
+
+# Model weights
+**/*.pth
+**/*.onnx
+
+pretrained_weights/*.md
+pretrained_weights/docs
+pretrained_weights/liveportrait
+pretrained_weights/liveportrait_animals
+
+# Ipython notebook
+*.ipynb
+
+# Temporary files or benchmark resources
+animations/*
+tmp/*
+.vscode/launch.json
+**/*.DS_Store
+gradio_temp/**
+
+# Windows dependencies
+ffmpeg/
+LivePortrait_env/
+
+# XPose build files
+src/utils/dependencies/XPose/models/UniPose/ops/build
+src/utils/dependencies/XPose/models/UniPose/ops/dist
+src/utils/dependencies/XPose/models/UniPose/ops/MultiScaleDeformableAttention.egg-info

ckpts/LivePortrait/.vscode/settings.json

@@ -0,0 +1,19 @@
+{
+    "[python]": {
+        "editor.tabSize": 4
+    },
+    "files.eol": "\n",
+    "files.insertFinalNewline": true,
+    "files.trimFinalNewlines": true,
+    "files.trimTrailingWhitespace": true,
+    "files.exclude": {
+        "**/.git": true,
+        "**/.svn": true,
+        "**/.hg": true,
+        "**/CVS": true,
+        "**/.DS_Store": true,
+        "**/Thumbs.db": true,
+        "**/*.crswap": true,
+        "**/__pycache__": true
+    }
+}

ckpts/LivePortrait/LICENSE

@@ -0,0 +1,30 @@
+MIT License
+
+Copyright (c) 2024 Kuaishou Visual Generation and Interaction Center
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+---
+
+The code of InsightFace is released under the MIT License.
+The models of InsightFace are for non-commercial research purposes only.
+
+If you want to use the LivePortrait project for commercial purposes, you 
+should remove and replace InsightFace’s detection models to fully comply with 
+the MIT license.

ckpts/LivePortrait/app.py

@@ -0,0 +1,493 @@
+# coding: utf-8
+
+"""
+The entrance of the gradio for human
+"""
+
+import os
+import tyro
+import subprocess
+import gradio as gr
+import os.path as osp
+from src.utils.helper import load_description
+from src.gradio_pipeline import GradioPipeline
+from src.config.crop_config import CropConfig
+from src.config.argument_config import ArgumentConfig
+from src.config.inference_config import InferenceConfig
+
+
+def partial_fields(target_class, kwargs):
+    return target_class(**{k: v for k, v in kwargs.items() if hasattr(target_class, k)})
+
+
+def fast_check_ffmpeg():
+    try:
+        subprocess.run(["ffmpeg", "-version"], capture_output=True, check=True)
+        return True
+    except:
+        return False
+
+
+# set tyro theme
+tyro.extras.set_accent_color("bright_cyan")
+args = tyro.cli(ArgumentConfig)
+
+ffmpeg_dir = os.path.join(os.getcwd(), "ffmpeg")
+if osp.exists(ffmpeg_dir):
+    os.environ["PATH"] += (os.pathsep + ffmpeg_dir)
+
+if not fast_check_ffmpeg():
+    raise ImportError(
+        "FFmpeg is not installed. Please install FFmpeg (including ffmpeg and ffprobe) before running this script. https://ffmpeg.org/download.html"
+    )
+# specify configs for inference
+inference_cfg = partial_fields(InferenceConfig, args.__dict__)  # use attribute of args to initial InferenceConfig
+crop_cfg = partial_fields(CropConfig, args.__dict__)  # use attribute of args to initial CropConfig
+# global_tab_selection = None
+
+gradio_pipeline = GradioPipeline(
+    inference_cfg=inference_cfg,
+    crop_cfg=crop_cfg,
+    args=args
+)
+
+if args.gradio_temp_dir not in (None, ''):
+    os.environ["GRADIO_TEMP_DIR"] = args.gradio_temp_dir
+    os.makedirs(args.gradio_temp_dir, exist_ok=True)
+
+
+def gpu_wrapped_execute_video(*args, **kwargs):
+    return gradio_pipeline.execute_video(*args, **kwargs)
+
+
+def gpu_wrapped_execute_image_retargeting(*args, **kwargs):
+    return gradio_pipeline.execute_image_retargeting(*args, **kwargs)
+
+
+def gpu_wrapped_execute_video_retargeting(*args, **kwargs):
+    return gradio_pipeline.execute_video_retargeting(*args, **kwargs)
+
+
+def reset_sliders(*args, **kwargs):
+    return 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2.5, True, True
+
+
+# assets
+title_md = "assets/gradio/gradio_title.md"
+example_portrait_dir = "assets/examples/source"
+example_video_dir = "assets/examples/driving"
+data_examples_i2v = [
+    [osp.join(example_portrait_dir, "s9.jpg"), osp.join(example_video_dir, "d0.mp4"), True, True, True, False],
+    [osp.join(example_portrait_dir, "s6.jpg"), osp.join(example_video_dir, "d0.mp4"), True, True, True, False],
+    [osp.join(example_portrait_dir, "s10.jpg"), osp.join(example_video_dir, "d0.mp4"), True, True, True, False],
+    [osp.join(example_portrait_dir, "s5.jpg"), osp.join(example_video_dir, "d18.mp4"), True, True, True, False],
+    [osp.join(example_portrait_dir, "s7.jpg"), osp.join(example_video_dir, "d19.mp4"), True, True, True, False],
+    [osp.join(example_portrait_dir, "s2.jpg"), osp.join(example_video_dir, "d13.mp4"), True, True, True, True],
+]
+data_examples_v2v = [
+    [osp.join(example_portrait_dir, "s13.mp4"), osp.join(example_video_dir, "d0.mp4"), True, True, True, False, 3e-7],
+    # [osp.join(example_portrait_dir, "s14.mp4"), osp.join(example_video_dir, "d18.mp4"), True, True, True, False, False, 3e-7],
+    # [osp.join(example_portrait_dir, "s15.mp4"), osp.join(example_video_dir, "d19.mp4"), True, True, True, False, False, 3e-7],
+    [osp.join(example_portrait_dir, "s18.mp4"), osp.join(example_video_dir, "d6.mp4"), True, True, True, False, 3e-7],
+    # [osp.join(example_portrait_dir, "s19.mp4"), osp.join(example_video_dir, "d6.mp4"), True, True, True, False, False, 3e-7],
+    [osp.join(example_portrait_dir, "s20.mp4"), osp.join(example_video_dir, "d0.mp4"), True, True, True, False, 3e-7],
+]
+#################### interface logic ####################
+
+# Define components first
+retargeting_source_scale = gr.Number(minimum=1.8, maximum=3.2, value=2.5, step=0.05, label="crop scale")
+video_retargeting_source_scale = gr.Number(minimum=1.8, maximum=3.2, value=2.3, step=0.05, label="crop scale")
+driving_smooth_observation_variance_retargeting = gr.Number(value=3e-6, label="motion smooth strength", minimum=1e-11, maximum=1e-2, step=1e-8)
+video_retargeting_silence = gr.Checkbox(value=False, label="keeping the lip silent")
+eye_retargeting_slider = gr.Slider(minimum=0, maximum=0.8, step=0.01, label="target eyes-open ratio")
+lip_retargeting_slider = gr.Slider(minimum=0, maximum=0.8, step=0.01, label="target lip-open ratio")
+video_lip_retargeting_slider = gr.Slider(minimum=0, maximum=0.8, step=0.01, label="target lip-open ratio")
+head_pitch_slider = gr.Slider(minimum=-15.0, maximum=15.0, value=0, step=1, label="relative pitch")
+head_yaw_slider = gr.Slider(minimum=-25, maximum=25, value=0, step=1, label="relative yaw")
+head_roll_slider = gr.Slider(minimum=-15.0, maximum=15.0, value=0, step=1, label="relative roll")
+mov_x = gr.Slider(minimum=-0.19, maximum=0.19, value=0.0, step=0.01, label="x-axis movement")
+mov_y = gr.Slider(minimum=-0.19, maximum=0.19, value=0.0, step=0.01, label="y-axis movement")
+mov_z = gr.Slider(minimum=0.9, maximum=1.2, value=1.0, step=0.01, label="z-axis movement")
+lip_variation_zero = gr.Slider(minimum=-0.09, maximum=0.09, value=0, step=0.01, label="pouting")
+lip_variation_one = gr.Slider(minimum=-20.0, maximum=15.0, value=0, step=0.01, label="pursing 😐")
+lip_variation_two = gr.Slider(minimum=0.0, maximum=15.0, value=0, step=0.01, label="grin 😁")
+lip_variation_three = gr.Slider(minimum=-90.0, maximum=120.0, value=0, step=1.0, label="lip close <-> open")
+smile = gr.Slider(minimum=-0.3, maximum=1.3, value=0, step=0.01, label="smile 😄")
+wink = gr.Slider(minimum=0, maximum=39, value=0, step=0.01, label="wink 😉")
+eyebrow = gr.Slider(minimum=-30, maximum=30, value=0, step=0.01, label="eyebrow 🤨")
+eyeball_direction_x = gr.Slider(minimum=-30.0, maximum=30.0, value=0, step=0.01, label="eye gaze (horizontal) 👀")
+eyeball_direction_y = gr.Slider(minimum=-63.0, maximum=63.0, value=0, step=0.01, label="eye gaze (vertical) 🙄")
+retargeting_input_image = gr.Image(type="filepath")
+retargeting_input_video = gr.Video()
+output_image = gr.Image(type="numpy")
+output_image_paste_back = gr.Image(type="numpy")
+retargeting_output_image = gr.Image(type="numpy")
+retargeting_output_image_paste_back = gr.Image(type="numpy")
+output_video = gr.Video(autoplay=False)
+output_video_paste_back = gr.Video(autoplay=False)
+
+with gr.Blocks(theme=gr.themes.Soft(font=[gr.themes.GoogleFont("Plus Jakarta Sans")])) as demo:
+    gr.HTML(load_description(title_md))
+
+    gr.Markdown(load_description("assets/gradio/gradio_description_upload.md"))
+    with gr.Row():
+        with gr.Column():
+            with gr.Tabs():
+                with gr.TabItem("🖼️ Source Image") as tab_image:
+                    with gr.Accordion(open=True, label="Source Image"):
+                        source_image_input = gr.Image(type="filepath")
+                        gr.Examples(
+                            examples=[
+                                [osp.join(example_portrait_dir, "s9.jpg")],
+                                [osp.join(example_portrait_dir, "s6.jpg")],
+                                [osp.join(example_portrait_dir, "s10.jpg")],
+                                [osp.join(example_portrait_dir, "s5.jpg")],
+                                [osp.join(example_portrait_dir, "s7.jpg")],
+                                [osp.join(example_portrait_dir, "s12.jpg")],
+                                [osp.join(example_portrait_dir, "s22.jpg")],
+                                [osp.join(example_portrait_dir, "s23.jpg")],
+                            ],
+                            inputs=[source_image_input],
+                            cache_examples=False,
+                        )
+
+                with gr.TabItem("🎞️ Source Video") as tab_video:
+                    with gr.Accordion(open=True, label="Source Video"):
+                        source_video_input = gr.Video()
+                        gr.Examples(
+                            examples=[
+                                [osp.join(example_portrait_dir, "s13.mp4")],
+                                # [osp.join(example_portrait_dir, "s14.mp4")],
+                                # [osp.join(example_portrait_dir, "s15.mp4")],
+                                [osp.join(example_portrait_dir, "s18.mp4")],
+                                # [osp.join(example_portrait_dir, "s19.mp4")],
+                                [osp.join(example_portrait_dir, "s20.mp4")],
+                            ],
+                            inputs=[source_video_input],
+                            cache_examples=False,
+                        )
+
+                tab_selection = gr.Textbox(visible=False)
+                tab_image.select(lambda: "Image", None, tab_selection)
+                tab_video.select(lambda: "Video", None, tab_selection)
+            with gr.Accordion(open=True, label="Cropping Options for Source Image or Video"):
+                with gr.Row():
+                    flag_do_crop_input = gr.Checkbox(value=True, label="do crop (source)")
+                    scale = gr.Number(value=2.3, label="source crop scale", minimum=1.8, maximum=3.2, step=0.05)
+                    vx_ratio = gr.Number(value=0.0, label="source crop x", minimum=-0.5, maximum=0.5, step=0.01)
+                    vy_ratio = gr.Number(value=-0.125, label="source crop y", minimum=-0.5, maximum=0.5, step=0.01)
+
+        with gr.Column():
+            with gr.Tabs():
+                with gr.TabItem("🎞️ Driving Video") as v_tab_video:
+                    with gr.Accordion(open=True, label="Driving Video"):
+                        driving_video_input = gr.Video()
+                        gr.Examples(
+                            examples=[
+                                [osp.join(example_video_dir, "d0.mp4")],
+                                [osp.join(example_video_dir, "d18.mp4")],
+                                [osp.join(example_video_dir, "d19.mp4")],
+                                [osp.join(example_video_dir, "d14.mp4")],
+                                [osp.join(example_video_dir, "d6.mp4")],
+                                [osp.join(example_video_dir, "d20.mp4")],
+                            ],
+                            inputs=[driving_video_input],
+                            cache_examples=False,
+                        )
+                with gr.TabItem("🖼️ Driving Image") as v_tab_image:
+                    with gr.Accordion(open=True, label="Driving Image"):
+                        driving_image_input = gr.Image(type="filepath")
+                        gr.Examples(
+                            examples=[
+                                [osp.join(example_video_dir, "d30.jpg")],
+                                [osp.join(example_video_dir, "d9.jpg")],
+                                [osp.join(example_video_dir, "d19.jpg")],
+                                [osp.join(example_video_dir, "d8.jpg")],
+                                [osp.join(example_video_dir, "d12.jpg")],
+                                [osp.join(example_video_dir, "d38.jpg")],
+                            ],
+                            inputs=[driving_image_input],
+                            cache_examples=False,
+                        )
+
+                with gr.TabItem("📁 Driving Pickle") as v_tab_pickle:
+                    with gr.Accordion(open=True, label="Driving Pickle"):
+                        driving_video_pickle_input = gr.File(type="filepath", file_types=[".pkl"])
+                        gr.Examples(
+                            examples=[
+                                [osp.join(example_video_dir, "d1.pkl")],
+                                [osp.join(example_video_dir, "d2.pkl")],
+                                [osp.join(example_video_dir, "d5.pkl")],
+                                [osp.join(example_video_dir, "d7.pkl")],
+                                [osp.join(example_video_dir, "d8.pkl")],
+                            ],
+                            inputs=[driving_video_pickle_input],
+                            cache_examples=False,
+                        )
+
+                v_tab_selection = gr.Textbox(visible=False)
+                v_tab_video.select(lambda: "Video", None, v_tab_selection)
+                v_tab_image.select(lambda: "Image", None, v_tab_selection)
+                v_tab_pickle.select(lambda: "Pickle", None, v_tab_selection)
+            # with gr.Accordion(open=False, label="Animation Instructions"):
+                # gr.Markdown(load_description("assets/gradio/gradio_description_animation.md"))
+            with gr.Accordion(open=True, label="Cropping Options for Driving Video"):
+                with gr.Row():
+                    flag_crop_driving_video_input = gr.Checkbox(value=False, label="do crop (driving)")
+                    scale_crop_driving_video = gr.Number(value=2.2, label="driving crop scale", minimum=1.8, maximum=3.2, step=0.05)
+                    vx_ratio_crop_driving_video = gr.Number(value=0.0, label="driving crop x", minimum=-0.5, maximum=0.5, step=0.01)
+                    vy_ratio_crop_driving_video = gr.Number(value=-0.1, label="driving crop y", minimum=-0.5, maximum=0.5, step=0.01)
+
+    with gr.Row():
+        with gr.Accordion(open=True, label="Animation Options"):
+            with gr.Row():
+                flag_normalize_lip = gr.Checkbox(value=False, label="normalize lip")
+                flag_relative_input = gr.Checkbox(value=True, label="relative motion")
+                flag_remap_input = gr.Checkbox(value=True, label="paste-back")
+                flag_stitching_input = gr.Checkbox(value=True, label="stitching")
+                animation_region = gr.Radio(["exp", "pose", "lip", "eyes", "all"], value="all", label="animation region")
+                driving_option_input = gr.Radio(['expression-friendly', 'pose-friendly'], value="expression-friendly", label="driving option (i2v)")
+                driving_multiplier = gr.Number(value=1.0, label="driving multiplier (i2v)", minimum=0.0, maximum=2.0, step=0.02)
+                driving_smooth_observation_variance = gr.Number(value=3e-7, label="motion smooth strength (v2v)", minimum=1e-11, maximum=1e-2, step=1e-8)
+
+    gr.Markdown(load_description("assets/gradio/gradio_description_animate_clear.md"))
+    with gr.Row():
+        process_button_animation = gr.Button("🚀 Animate", variant="primary")
+    with gr.Row():
+        with gr.Column():
+            output_video_i2v = gr.Video(autoplay=False, label="The animated video in the original image space")
+        with gr.Column():
+            output_video_concat_i2v = gr.Video(autoplay=False, label="The animated video")
+    with gr.Row():
+        with gr.Column():
+            output_image_i2i = gr.Image(type="numpy", label="The animated image in the original image space", visible=False)
+        with gr.Column():
+            output_image_concat_i2i = gr.Image(type="numpy", label="The animated image", visible=False)
+    with gr.Row():
+        process_button_reset = gr.ClearButton([source_image_input, source_video_input, driving_video_pickle_input, driving_video_input, driving_image_input, output_video_i2v, output_video_concat_i2v, output_image_i2i, output_image_concat_i2i], value="🧹 Clear")
+
+    with gr.Row():
+        # Examples
+        gr.Markdown("## You could also choose the examples below by one click ⬇️")
+    with gr.Row():
+        with gr.Tabs():
+            with gr.TabItem("🖼️ Portrait Animation"):
+                gr.Examples(
+                    examples=data_examples_i2v,
+                    fn=gpu_wrapped_execute_video,
+                    inputs=[
+                        source_image_input,
+                        driving_video_input,
+                        flag_relative_input,
+                        flag_do_crop_input,
+                        flag_remap_input,
+                        flag_crop_driving_video_input,
+                    ],
+                    outputs=[output_image, output_image_paste_back],
+                    examples_per_page=len(data_examples_i2v),
+                    cache_examples=False,
+                )
+            with gr.TabItem("🎞️ Portrait Video Editing"):
+                gr.Examples(
+                    examples=data_examples_v2v,
+                    fn=gpu_wrapped_execute_video,
+                    inputs=[
+                        source_video_input,
+                        driving_video_input,
+                        flag_relative_input,
+                        flag_do_crop_input,
+                        flag_remap_input,
+                        flag_crop_driving_video_input,
+                        driving_smooth_observation_variance,
+                    ],
+                    outputs=[output_image, output_image_paste_back],
+                    examples_per_page=len(data_examples_v2v),
+                    cache_examples=False,
+                )
+
+    # Retargeting Image
+    gr.Markdown(load_description("assets/gradio/gradio_description_retargeting.md"), visible=True)
+    with gr.Row(visible=True):
+        flag_do_crop_input_retargeting_image = gr.Checkbox(value=True, label="do crop (source)")
+        flag_stitching_retargeting_input = gr.Checkbox(value=True, label="stitching")
+        retargeting_source_scale.render()
+        eye_retargeting_slider.render()
+        lip_retargeting_slider.render()
+    with gr.Row(visible=True):
+        with gr.Column():
+            with gr.Accordion(open=True, label="Facial movement sliders"):
+                with gr.Row(visible=True):
+                    head_pitch_slider.render()
+                    head_yaw_slider.render()
+                    head_roll_slider.render()
+                with gr.Row(visible=True):
+                    mov_x.render()
+                    mov_y.render()
+                    mov_z.render()
+        with gr.Column():
+            with gr.Accordion(open=True, label="Facial expression sliders"):
+                with gr.Row(visible=True):
+                    lip_variation_zero.render()
+                    lip_variation_one.render()
+                    lip_variation_two.render()
+                with gr.Row(visible=True):
+                    lip_variation_three.render()
+                    smile.render()
+                    wink.render()
+                with gr.Row(visible=True):
+                    eyebrow.render()
+                    eyeball_direction_x.render()
+                    eyeball_direction_y.render()
+    with gr.Row(visible=True):
+        reset_button = gr.Button("🔄 Reset")
+        reset_button.click(
+            fn=reset_sliders,
+            inputs=None,
+            outputs=[
+                head_pitch_slider, head_yaw_slider, head_roll_slider, mov_x, mov_y, mov_z,
+                lip_variation_zero, lip_variation_one, lip_variation_two, lip_variation_three, smile, wink, eyebrow, eyeball_direction_x, eyeball_direction_y,
+                retargeting_source_scale, flag_stitching_retargeting_input, flag_do_crop_input_retargeting_image
+            ]
+        )
+    with gr.Row(visible=True):
+        with gr.Column():
+            with gr.Accordion(open=True, label="Retargeting Image Input"):
+                retargeting_input_image.render()
+                gr.Examples(
+                    examples=[
+                        [osp.join(example_portrait_dir, "s9.jpg")],
+                        [osp.join(example_portrait_dir, "s6.jpg")],
+                        [osp.join(example_portrait_dir, "s10.jpg")],
+                        [osp.join(example_portrait_dir, "s5.jpg")],
+                        [osp.join(example_portrait_dir, "s7.jpg")],
+                        [osp.join(example_portrait_dir, "s12.jpg")],
+                        [osp.join(example_portrait_dir, "s22.jpg")],
+                        # [osp.join(example_portrait_dir, "s23.jpg")],
+                        [osp.join(example_portrait_dir, "s42.jpg")],
+                    ],
+                    inputs=[retargeting_input_image],
+                    cache_examples=False,
+                )
+        with gr.Column():
+            with gr.Accordion(open=True, label="Retargeting Result"):
+                retargeting_output_image.render()
+        with gr.Column():
+            with gr.Accordion(open=True, label="Paste-back Result"):
+                retargeting_output_image_paste_back.render()
+    with gr.Row(visible=True):
+        process_button_reset_retargeting = gr.ClearButton(
+            [
+                retargeting_input_image,
+                retargeting_output_image,
+                retargeting_output_image_paste_back,
+            ],
+            value="🧹 Clear"
+        )
+
+    # Retargeting Video
+    gr.Markdown(load_description("assets/gradio/gradio_description_retargeting_video.md"), visible=True)
+    with gr.Row(visible=True):
+        flag_do_crop_input_retargeting_video = gr.Checkbox(value=True, label="do crop (source)")
+        video_retargeting_source_scale.render()
+        video_lip_retargeting_slider.render()
+        driving_smooth_observation_variance_retargeting.render()
+        video_retargeting_silence.render()
+    with gr.Row(visible=True):
+        process_button_retargeting_video = gr.Button("🚗 Retargeting Video", variant="primary")
+    with gr.Row(visible=True):
+        with gr.Column():
+            with gr.Accordion(open=True, label="Retargeting Video Input"):
+                retargeting_input_video.render()
+                gr.Examples(
+                    examples=[
+                        [osp.join(example_portrait_dir, "s13.mp4")],
+                        # [osp.join(example_portrait_dir, "s18.mp4")],
+                        # [osp.join(example_portrait_dir, "s20.mp4")],
+                        [osp.join(example_portrait_dir, "s29.mp4")],
+                        [osp.join(example_portrait_dir, "s32.mp4")],
+                        [osp.join(example_video_dir, "d3.mp4")],
+                    ],
+                    inputs=[retargeting_input_video],
+                    cache_examples=False,
+                )
+        with gr.Column():
+            with gr.Accordion(open=True, label="Retargeting Result"):
+                output_video.render()
+        with gr.Column():
+            with gr.Accordion(open=True, label="Paste-back Result"):
+                output_video_paste_back.render()
+    with gr.Row(visible=True):
+        process_button_reset_retargeting = gr.ClearButton(
+            [
+                video_lip_retargeting_slider,
+                retargeting_input_video,
+                output_video,
+                output_video_paste_back
+            ],
+            value="🧹 Clear"
+        )
+
+    # binding functions for buttons
+    process_button_animation.click(
+        fn=gpu_wrapped_execute_video,
+        inputs=[
+            source_image_input,
+            source_video_input,
+            driving_video_input,
+            driving_image_input,
+            driving_video_pickle_input,
+            flag_normalize_lip,
+            flag_relative_input,
+            flag_do_crop_input,
+            flag_remap_input,
+            flag_stitching_input,
+            animation_region,
+            driving_option_input,
+            driving_multiplier,
+            flag_crop_driving_video_input,
+            scale,
+            vx_ratio,
+            vy_ratio,
+            scale_crop_driving_video,
+            vx_ratio_crop_driving_video,
+            vy_ratio_crop_driving_video,
+            driving_smooth_observation_variance,
+            tab_selection,
+            v_tab_selection,
+        ],
+        outputs=[output_video_i2v, output_video_i2v, output_video_concat_i2v, output_video_concat_i2v, output_image_i2i, output_image_i2i, output_image_concat_i2i, output_image_concat_i2i],
+        show_progress=True
+    )
+
+
+    retargeting_input_image.change(
+        fn=gradio_pipeline.init_retargeting_image,
+        inputs=[retargeting_source_scale, eye_retargeting_slider, lip_retargeting_slider, retargeting_input_image],
+        outputs=[eye_retargeting_slider, lip_retargeting_slider]
+    )
+
+    sliders = [eye_retargeting_slider, lip_retargeting_slider, head_pitch_slider, head_yaw_slider, head_roll_slider, mov_x, mov_y, mov_z, lip_variation_zero, lip_variation_one, lip_variation_two, lip_variation_three, smile, wink, eyebrow, eyeball_direction_x, eyeball_direction_y]
+    for slider in sliders:
+        # NOTE: gradio >= 4.0.0 may cause slow response
+        slider.change(
+            fn=gpu_wrapped_execute_image_retargeting,
+            inputs=[
+                eye_retargeting_slider, lip_retargeting_slider, head_pitch_slider, head_yaw_slider, head_roll_slider, mov_x, mov_y, mov_z,
+                lip_variation_zero, lip_variation_one, lip_variation_two, lip_variation_three, smile, wink, eyebrow, eyeball_direction_x, eyeball_direction_y,
+                retargeting_input_image, retargeting_source_scale, flag_stitching_retargeting_input, flag_do_crop_input_retargeting_image
+            ],
+            outputs=[retargeting_output_image, retargeting_output_image_paste_back],
+        )
+
+    process_button_retargeting_video.click(
+        fn=gpu_wrapped_execute_video_retargeting,
+        inputs=[video_lip_retargeting_slider, retargeting_input_video, video_retargeting_source_scale, driving_smooth_observation_variance_retargeting, video_retargeting_silence, flag_do_crop_input_retargeting_video],
+        outputs=[output_video, output_video_paste_back],
+        show_progress=True
+    )
+
+demo.launch(
+    server_port=args.server_port,
+    share=args.share,
+    server_name=args.server_name
+)

ckpts/LivePortrait/app_animals.py

@@ -0,0 +1,249 @@
+# coding: utf-8
+
+"""
+The entrance of the gradio for animal
+"""
+
+import os
+import tyro
+import subprocess
+import gradio as gr
+import os.path as osp
+from src.utils.helper import load_description
+from src.gradio_pipeline import GradioPipelineAnimal
+from src.config.crop_config import CropConfig
+from src.config.argument_config import ArgumentConfig
+from src.config.inference_config import InferenceConfig
+
+
+def partial_fields(target_class, kwargs):
+    return target_class(**{k: v for k, v in kwargs.items() if hasattr(target_class, k)})
+
+
+def fast_check_ffmpeg():
+    try:
+        subprocess.run(["ffmpeg", "-version"], capture_output=True, check=True)
+        return True
+    except:
+        return False
+
+
+# set tyro theme
+tyro.extras.set_accent_color("bright_cyan")
+args = tyro.cli(ArgumentConfig)
+
+ffmpeg_dir = os.path.join(os.getcwd(), "ffmpeg")
+if osp.exists(ffmpeg_dir):
+    os.environ["PATH"] += (os.pathsep + ffmpeg_dir)
+
+if not fast_check_ffmpeg():
+    raise ImportError(
+        "FFmpeg is not installed. Please install FFmpeg (including ffmpeg and ffprobe) before running this script. https://ffmpeg.org/download.html"
+    )
+# specify configs for inference
+inference_cfg = partial_fields(InferenceConfig, args.__dict__)  # use attribute of args to initial InferenceConfig
+crop_cfg = partial_fields(CropConfig, args.__dict__)  # use attribute of args to initial CropConfig
+
+gradio_pipeline_animal: GradioPipelineAnimal = GradioPipelineAnimal(
+    inference_cfg=inference_cfg,
+    crop_cfg=crop_cfg,
+    args=args
+)
+
+if args.gradio_temp_dir not in (None, ''):
+    os.environ["GRADIO_TEMP_DIR"] = args.gradio_temp_dir
+    os.makedirs(args.gradio_temp_dir, exist_ok=True)
+
+def gpu_wrapped_execute_video(*args, **kwargs):
+    return gradio_pipeline_animal.execute_video(*args, **kwargs)
+
+
+# assets
+title_md = "assets/gradio/gradio_title.md"
+example_portrait_dir = "assets/examples/source"
+example_video_dir = "assets/examples/driving"
+data_examples_i2v = [
+    [osp.join(example_portrait_dir, "s41.jpg"), osp.join(example_video_dir, "d3.mp4"), True, False, False, False],
+    [osp.join(example_portrait_dir, "s40.jpg"), osp.join(example_video_dir, "d6.mp4"), True, False, False, False],
+    [osp.join(example_portrait_dir, "s25.jpg"), osp.join(example_video_dir, "d19.mp4"), True, False, False, False],
+]
+data_examples_i2v_pickle = [
+    [osp.join(example_portrait_dir, "s25.jpg"), osp.join(example_video_dir, "wink.pkl"), True, False, False, False],
+    [osp.join(example_portrait_dir, "s40.jpg"), osp.join(example_video_dir, "talking.pkl"), True, False, False, False],
+    [osp.join(example_portrait_dir, "s41.jpg"), osp.join(example_video_dir, "aggrieved.pkl"), True, False, False, False],
+]
+#################### interface logic ####################
+
+# Define components first
+output_image = gr.Image(type="numpy")
+output_image_paste_back = gr.Image(type="numpy")
+output_video_i2v = gr.Video(autoplay=False)
+output_video_concat_i2v = gr.Video(autoplay=False)
+output_video_i2v_gif = gr.Image(type="numpy")
+
+
+with gr.Blocks(theme=gr.themes.Soft(font=[gr.themes.GoogleFont("Plus Jakarta Sans")])) as demo:
+    gr.HTML(load_description(title_md))
+
+    gr.Markdown(load_description("assets/gradio/gradio_description_upload_animal.md"))
+    with gr.Row():
+        with gr.Column():
+            with gr.Accordion(open=True, label="🐱 Source Animal Image"):
+                source_image_input = gr.Image(type="filepath")
+                gr.Examples(
+                    examples=[
+                        [osp.join(example_portrait_dir, "s25.jpg")],
+                        [osp.join(example_portrait_dir, "s30.jpg")],
+                        [osp.join(example_portrait_dir, "s31.jpg")],
+                        [osp.join(example_portrait_dir, "s32.jpg")],
+                        [osp.join(example_portrait_dir, "s33.jpg")],
+                        [osp.join(example_portrait_dir, "s39.jpg")],
+                        [osp.join(example_portrait_dir, "s40.jpg")],
+                        [osp.join(example_portrait_dir, "s41.jpg")],
+                        [osp.join(example_portrait_dir, "s38.jpg")],
+                        [osp.join(example_portrait_dir, "s36.jpg")],
+                    ],
+                    inputs=[source_image_input],
+                    cache_examples=False,
+                )
+
+            with gr.Accordion(open=True, label="Cropping Options for Source Image"):
+                with gr.Row():
+                    flag_do_crop_input = gr.Checkbox(value=True, label="do crop (source)")
+                    scale = gr.Number(value=2.3, label="source crop scale", minimum=1.8, maximum=3.2, step=0.05)
+                    vx_ratio = gr.Number(value=0.0, label="source crop x", minimum=-0.5, maximum=0.5, step=0.01)
+                    vy_ratio = gr.Number(value=-0.125, label="source crop y", minimum=-0.5, maximum=0.5, step=0.01)
+
+        with gr.Column():
+            with gr.Tabs():
+                with gr.TabItem("📁 Driving Pickle") as tab_pickle:
+                    with gr.Accordion(open=True, label="Driving Pickle"):
+                        driving_video_pickle_input = gr.File()
+                        gr.Examples(
+                            examples=[
+                                [osp.join(example_video_dir, "wink.pkl")],
+                                [osp.join(example_video_dir, "shy.pkl")],
+                                [osp.join(example_video_dir, "aggrieved.pkl")],
+                                [osp.join(example_video_dir, "open_lip.pkl")],
+                                [osp.join(example_video_dir, "laugh.pkl")],
+                                [osp.join(example_video_dir, "talking.pkl")],
+                                [osp.join(example_video_dir, "shake_face.pkl")],
+                            ],
+                            inputs=[driving_video_pickle_input],
+                            cache_examples=False,
+                        )
+                with gr.TabItem("🎞️ Driving Video") as tab_video:
+                    with gr.Accordion(open=True, label="Driving Video"):
+                        driving_video_input = gr.Video()
+                        gr.Examples(
+                            examples=[
+                                # [osp.join(example_video_dir, "d0.mp4")],
+                                # [osp.join(example_video_dir, "d18.mp4")],
+                                [osp.join(example_video_dir, "d19.mp4")],
+                                [osp.join(example_video_dir, "d14.mp4")],
+                                [osp.join(example_video_dir, "d6.mp4")],
+                                [osp.join(example_video_dir, "d3.mp4")],
+                            ],
+                            inputs=[driving_video_input],
+                            cache_examples=False,
+                        )
+
+                    tab_selection = gr.Textbox(visible=False)
+                    tab_pickle.select(lambda: "Pickle", None, tab_selection)
+                    tab_video.select(lambda: "Video", None, tab_selection)
+            with gr.Accordion(open=True, label="Cropping Options for Driving Video"):
+                with gr.Row():
+                    flag_crop_driving_video_input = gr.Checkbox(value=False, label="do crop (driving)")
+                    scale_crop_driving_video = gr.Number(value=2.2, label="driving crop scale", minimum=1.8, maximum=3.2, step=0.05)
+                    vx_ratio_crop_driving_video = gr.Number(value=0.0, label="driving crop x", minimum=-0.5, maximum=0.5, step=0.01)
+                    vy_ratio_crop_driving_video = gr.Number(value=-0.1, label="driving crop y", minimum=-0.5, maximum=0.5, step=0.01)
+
+    with gr.Row():
+        with gr.Accordion(open=False, label="Animation Options"):
+            with gr.Row():
+                flag_stitching = gr.Checkbox(value=False, label="stitching (not recommended)")
+                flag_remap_input = gr.Checkbox(value=False, label="paste-back (not recommended)")
+                driving_multiplier = gr.Number(value=1.0, label="driving multiplier", minimum=0.0, maximum=2.0, step=0.02)
+
+    gr.Markdown(load_description("assets/gradio/gradio_description_animate_clear.md"))
+    with gr.Row():
+        process_button_animation = gr.Button("🚀 Animate", variant="primary")
+    with gr.Row():
+        with gr.Column():
+            with gr.Accordion(open=True, label="The animated video in the cropped image space"):
+                output_video_i2v.render()
+        with gr.Column():
+            with gr.Accordion(open=True, label="The animated gif in the cropped image space"):
+                output_video_i2v_gif.render()
+        with gr.Column():
+            with gr.Accordion(open=True, label="The animated video"):
+                output_video_concat_i2v.render()
+    with gr.Row():
+        process_button_reset = gr.ClearButton([source_image_input, driving_video_input, output_video_i2v, output_video_concat_i2v, output_video_i2v_gif], value="🧹 Clear")
+
+    with gr.Row():
+        # Examples
+        gr.Markdown("## You could also choose the examples below by one click ⬇️")
+    with gr.Row():
+        with gr.Tabs():
+            with gr.TabItem("📁 Driving Pickle") as tab_video:
+                gr.Examples(
+                    examples=data_examples_i2v_pickle,
+                    fn=gpu_wrapped_execute_video,
+                    inputs=[
+                        source_image_input,
+                        driving_video_pickle_input,
+                        flag_do_crop_input,
+                        flag_stitching,
+                        flag_remap_input,
+                        flag_crop_driving_video_input,
+                    ],
+                    outputs=[output_image, output_image_paste_back, output_video_i2v_gif],
+                    examples_per_page=len(data_examples_i2v_pickle),
+                    cache_examples=False,
+                )
+            with gr.TabItem("🎞️ Driving Video") as tab_video:
+                gr.Examples(
+                    examples=data_examples_i2v,
+                    fn=gpu_wrapped_execute_video,
+                    inputs=[
+                        source_image_input,
+                        driving_video_input,
+                        flag_do_crop_input,
+                        flag_stitching,
+                        flag_remap_input,
+                        flag_crop_driving_video_input,
+                    ],
+                    outputs=[output_image, output_image_paste_back, output_video_i2v_gif],
+                    examples_per_page=len(data_examples_i2v),
+                    cache_examples=False,
+                )
+
+    process_button_animation.click(
+        fn=gpu_wrapped_execute_video,
+        inputs=[
+            source_image_input,
+            driving_video_input,
+            driving_video_pickle_input,
+            flag_do_crop_input,
+            flag_remap_input,
+            driving_multiplier,
+            flag_stitching,
+            flag_crop_driving_video_input,
+            scale,
+            vx_ratio,
+            vy_ratio,
+            scale_crop_driving_video,
+            vx_ratio_crop_driving_video,
+            vy_ratio_crop_driving_video,
+            tab_selection,
+        ],
+        outputs=[output_video_i2v, output_video_concat_i2v, output_video_i2v_gif],
+        show_progress=True
+    )
+
+demo.launch(
+    server_port=args.server_port,
+    share=args.share,
+    server_name=args.server_name
+)

ckpts/LivePortrait/assets/.gitignore

@@ -0,0 +1,2 @@
+examples/driving/*.pkl
+examples/driving/*_crop.mp4

ckpts/LivePortrait/assets/docs/changelog/2024-07-10.md

@@ -0,0 +1,22 @@
+## 2024/07/10
+
+**First, thank you all for your attention, support, sharing, and contributions to LivePortrait!** ❤️
+The popularity of LivePortrait has exceeded our expectations. If you encounter any issues or other problems and we do not respond promptly, please accept our apologies. We are still actively updating and improving this repository.
+
+### Updates
+
+- <strong>Audio and video concatenating: </strong> If the driving video contains audio, it will automatically be included in the generated video. Additionally, the generated video will maintain the same FPS as the driving video. If you run LivePortrait on Windows, you need to install `ffprobe` and `ffmpeg` exe, see issue [#94](https://github.com/KwaiVGI/LivePortrait/issues/94).
+
+- <strong>Driving video auto-cropping: </strong> Implemented automatic cropping for driving videos by tracking facial landmarks and calculating a global cropping box with a 1:1 aspect ratio. Alternatively, you can crop using video editing software or other tools to achieve a 1:1 ratio. Auto-cropping is not enbaled by default, you can specify it by `--flag_crop_driving_video`.
+
+- <strong>Motion template making: </strong> Added the ability to create motion templates to protect privacy. The motion template is a `.pkl` file that only contains the motions of the driving video. Theoretically, it is impossible to reconstruct the original face from the template. These motion templates can be used to generate videos without needing the original driving video. By default, the motion template will be generated and saved as a `.pkl` file with the same name as the driving video, e.g., `d0.mp4` -> `d0.pkl`. Once generated, you can specify it using the `-d` or `--driving` option.
+
+
+### About driving video
+
+- For a guide on using your own driving video, see the [driving video auto-cropping](https://github.com/KwaiVGI/LivePortrait/tree/main?tab=readme-ov-file#driving-video-auto-cropping) section.
+
+
+### Others
+
+- If you encounter a black box problem, disable half-precision inference by using `--no_flag_use_half_precision`, reported by issue [#40](https://github.com/KwaiVGI/LivePortrait/issues/40), [#48](https://github.com/KwaiVGI/LivePortrait/issues/48), [#62](https://github.com/KwaiVGI/LivePortrait/issues/62).

ckpts/LivePortrait/assets/docs/changelog/2024-07-19.md

@@ -0,0 +1,24 @@
+## 2024/07/19
+
+**Once again, we would like to express our heartfelt gratitude for your love, attention, and support for LivePortrait! 🎉**
+We are excited to announce the release of an implementation of Portrait Video Editing (aka v2v) today! Special thanks to the hard work of the LivePortrait team: [Dingyun Zhang](https://github.com/Mystery099), [Zhizhou Zhong](https://github.com/zzzweakman), and [Jianzhu Guo](https://github.com/cleardusk).
+
+### Updates
+
+- <strong>Portrait video editing (v2v):</strong> Implemented a version of Portrait Video Editing (aka v2v). Ensure you have `pykalman` package installed, which has been added in [`requirements_base.txt`](../../../requirements_base.txt). You can specify the source video using the `-s` or `--source` option,  adjust the temporal smoothness of motion with `--driving_smooth_observation_variance`, enable head pose motion transfer with `--flag_video_editing_head_rotation`, and ensure the eye-open scalar of each source frame matches the first source frame before animation with `--flag_source_video_eye_retargeting`.
+
+- <strong>More options in Gradio:</strong> We have upgraded the Gradio interface and added more options. These include `Cropping Options for Source Image or Video` and `Cropping Options for Driving Video`, providing greater flexibility and control.
+
+<p align="center">
+  <img src="../LivePortrait-Gradio-2024-07-19.jpg" alt="LivePortrait" width="800px">
+  <br>
+  The Gradio Interface for LivePortrait
+</p>
+
+
+### Community Contributions
+
+- **ONNX/TensorRT Versions of LivePortrait:** Explore optimized versions of LivePortrait for faster performance:
+  - [FasterLivePortrait](https://github.com/warmshao/FasterLivePortrait) by [warmshao](https://github.com/warmshao) ([#150](https://github.com/KwaiVGI/LivePortrait/issues/150))
+  - [Efficient-Live-Portrait](https://github.com/aihacker111/Efficient-Live-Portrait) by [aihacker111](https://github.com/aihacker111/Efficient-Live-Portrait) ([#126](https://github.com/KwaiVGI/LivePortrait/issues/126), [#142](https://github.com/KwaiVGI/LivePortrait/issues/142))
+- **LivePortrait with [X-Pose](https://github.com/IDEA-Research/X-Pose) Detection:** Check out [LivePortrait](https://github.com/ShiJiaying/LivePortrait) by [ShiJiaying](https://github.com/ShiJiaying) for enhanced detection capabilities using X-pose, see [#119](https://github.com/KwaiVGI/LivePortrait/issues/119).

ckpts/LivePortrait/assets/docs/changelog/2024-07-24.md

@@ -0,0 +1,12 @@
+## 2024/07/24
+
+### Updates
+
+- **Portrait pose editing:** You can change the `relative pitch`, `relative yaw`, and `relative roll` in the Gradio interface to adjust the pose of the source portrait.
+- **Detection threshold:** We have added a `--det_thresh` argument with a default value of 0.15 to increase recall, meaning more types of faces (e.g., monkeys, human-like) will be detected. You can set it to other values, e.g., 0.5, by using `python app.py --det_thresh 0.5`.
+
+<p align="center">
+  <img src="../pose-edit-2024-07-24.jpg" alt="LivePortrait" width="960px">
+  <br>
+  Pose Editing in the Gradio Interface
+</p>

ckpts/LivePortrait/assets/docs/changelog/2024-08-02.md

@@ -0,0 +1,75 @@
+## 2024/08/02
+
+<table class="center" style="width: 80%; margin-left: auto; margin-right: auto;">
+<tr>
+    <td style="text-align: center"><b>Animals Singing Dance Monkey 🎤</b></td>
+</tr>
+
+<tr>
+    <td style="border: none; text-align: center;">
+        <video controls loop src="https://github.com/user-attachments/assets/38d5b6e5-d29b-458d-9f2c-4dd52546cb41" muted="false" style="width: 60%;"></video>
+    </td>
+</tr>
+</table>
+
+
+🎉 We are excited to announce the release of a new version featuring animals mode, along with several other updates. Special thanks to the dedicated efforts of the LivePortrait team. 💪 We also provided an one-click installer for Windows users, checkout the details [here](./2024-08-05.md).
+
+### Updates on Animals mode
+We are pleased to announce the release of the animals mode, which is fine-tuned on approximately 230K frames of various animals (mostly cats and dogs). The trained weights have been updated in the `liveportrait_animals` subdirectory, available on [HuggingFace](https://huggingface.co/KwaiVGI/LivePortrait/tree/main/) or [Google Drive](https://drive.google.com/drive/u/0/folders/1UtKgzKjFAOmZkhNK-OYT0caJ_w2XAnib). You should [download the weights](https://github.com/KwaiVGI/LivePortrait?tab=readme-ov-file#2-download-pretrained-weights) before running. There are two ways to run this mode.
+
+> Please note that we have not trained the stitching and retargeting modules for the animals model due to several technical issues. _This may be addressed in future updates._ Therefore, we recommend **disabling stitching by setting the `--no_flag_stitching`** option when running the model. Additionally, `paste-back` is also not recommended.
+
+#### Install X-Pose
+We have chosen [X-Pose](https://github.com/IDEA-Research/X-Pose) as the keypoints detector for animals. This relies on `transformers==4.22.0` and `pillow>=10.2.0` (which are already updated in `requirements.txt`) and requires building an OP named `MultiScaleDeformableAttention`.
+
+Refer to the [PyTorch installation](https://github.com/KwaiVGI/LivePortrait?tab=readme-ov-file#for-linux-or-windows-users) for Linux and Windows users.
+
+
+Next, build the OP `MultiScaleDeformableAttention` by running:
+```bash
+cd src/utils/dependencies/XPose/models/UniPose/ops
+python setup.py build install
+cd - # this returns to the previous directory
+```
+
+To run the model, use the `inference_animals.py` script:
+```bash
+python inference_animals.py -s assets/examples/source/s39.jpg -d assets/examples/driving/wink.pkl --no_flag_stitching --driving_multiplier 1.75
+```
+
+Alternatively, you can use Gradio for a more user-friendly interface. Launch it with:
+```bash
+python app_animals.py # --server_port 8889 --server_name "0.0.0.0" --share
+```
+
+> [!WARNING]
+> [X-Pose](https://github.com/IDEA-Research/X-Pose) is only for Non-commercial Scientific Research Purposes, you should remove and replace it with other detectors if you use it for commercial purposes.
+
+### Updates on Humans mode
+
+- **Driving Options**: We have introduced an `expression-friendly` driving option to **reduce head wobbling**, now set as the default. While it may be less effective with large head poses, you can also select the `pose-friendly` option, which is the same as the previous version. This can be set using `--driving_option` or selected in the Gradio interface. Additionally, we added a `--driving_multiplier` option to adjust driving intensity, with a default value of 1, which can also be set in the Gradio interface.
+
+- **Retargeting Video in Gradio**: We have implemented a video retargeting feature. You can specify a `target lip-open ratio` to adjust the mouth movement in the source video. For instance, setting it to 0 will close the mouth in the source video 🤐.
+
+### Others
+
+- [**Poe supports LivePortrait**](https://poe.com/LivePortrait). Check out the news on [X](https://x.com/poe_platform/status/1816136105781256260).
+- [ComfyUI-LivePortraitKJ](https://github.com/kijai/ComfyUI-LivePortraitKJ) (1.1K 🌟) now includes MediaPipe as an alternative to InsightFace, ensuring the license remains under MIT and Apache 2.0.
+- [ComfyUI-AdvancedLivePortrait](https://github.com/PowerHouseMan/ComfyUI-AdvancedLivePortrait) features real-time portrait pose/expression editing and animation, and is registered with ComfyUI-Manager.
+
+
+
+**Below are some screenshots of the new features and improvements:**
+
+| ![The Gradio Interface of Animals Mode](../animals-mode-gradio-2024-08-02.jpg) |
+|:---:|
+| **The Gradio Interface of Animals Mode** |
+
+| ![Driving Options and Multiplier](../driving-option-multiplier-2024-08-02.jpg) |
+|:---:|
+| **Driving Options and Multiplier** |
+
+| ![The Feature of Retargeting Video](../retargeting-video-2024-08-02.jpg) |
+|:---:|
+| **The Feature of Retargeting Video** |

ckpts/LivePortrait/assets/docs/changelog/2024-08-05.md

@@ -0,0 +1,18 @@
+## One-click Windows Installer
+
+### Download the installer from HuggingFace
+```bash
+# !pip install -U "huggingface_hub[cli]"
+huggingface-cli download cleardusk/LivePortrait-Windows LivePortrait-Windows-v20240806.zip --local-dir ./
+```
+
+If you cannot access to Huggingface, you can use [hf-mirror](https://hf-mirror.com/) to download:
+```bash
+# !pip install -U "huggingface_hub[cli]"
+export HF_ENDPOINT=https://hf-mirror.com
+huggingface-cli download cleardusk/LivePortrait-Windows LivePortrait-Windows-v20240806.zip --local-dir ./
+```
+
+Alternatively, you can manually download it from the [HuggingFace](https://huggingface.co/cleardusk/LivePortrait-Windows/blob/main/LivePortrait-Windows-v20240806.zip) page.
+
+Then, simply unzip the package `LivePortrait-Windows-v20240806.zip` and double-click `run_windows_human.bat` for the Humans mode, or `run_windows_animal.bat` for the **Animals mode**.

ckpts/LivePortrait/assets/docs/changelog/2024-08-06.md

@@ -0,0 +1,9 @@
+## Precise Portrait Editing
+
+Inspired by [ComfyUI-AdvancedLivePortrait](https://github.com/PowerHouseMan/ComfyUI-AdvancedLivePortrait) ([@PowerHouseMan](https://github.com/PowerHouseMan)), we have implemented a version of Precise Portrait Editing in the Gradio interface. With each adjustment of the slider, the edited image updates in real-time. You can click the `🔄 Reset` button to reset all slider parameters. However, the performance may not be as fast as the ComfyUI plugin.
+
+<p align="center">
+  <img src="../editing-portrait-2024-08-06.jpg" alt="LivePortrait" width="960px">
+  <br>
+  Preciese Portrait Editing in the Gradio Interface
+</p>

ckpts/LivePortrait/assets/docs/changelog/2024-08-19.md

@@ -0,0 +1,65 @@
+## Image Driven and Regional Control
+
+<p align="center">
+  <img src="../image-driven-image-2024-08-19.jpg" alt="LivePortrait" width="512px">
+  <br>
+  <strong>Image Drives an Image</strong>
+</p>
+
+You can now **use an image as a driving signal** to drive the source image or video! Additionally, we **have refined the driving options to support expressions, pose, lips, eyes, or all** (all is consistent with the previous default method), which we name it regional control. The control is becoming more and more precise! 🎯
+
+> Please note that image-based driving or regional control may not perform well in certain cases. Feel free to try different options, and be patient. 😊
+
+> [!Note]
+> We recognize that the project now offers more options, which have become increasingly complex, but due to our limited team capacity and resources, we haven’t fully documented them yet. We ask for your understanding and will work to improve the documentation over time. Contributions via PRs are welcome! If anyone is considering donating or sponsoring, feel free to leave a message in the GitHub Issues or Discussions. We will set up a payment account to reward the team members or support additional efforts in maintaining the project. 💖
+
+
+### CLI Usage
+It's very simple to use an image as a driving reference. Just set the `-d` argument to the driving image:
+
+```bash
+python inference.py -s assets/examples/source/s5.jpg -d assets/examples/driving/d30.jpg
+```
+
+To change the `animation_region` option, you can use the `--animation_region` argument to `exp`, `pose`, `lip`, `eyes`, or `all`. For example, to only drive the lip region, you can run by:
+
+```bash
+# only driving the lip region
+python inference.py -s assets/examples/source/s5.jpg -d assets/examples/driving/d0.mp4 --animation_region lip
+```
+
+### Gradio Interface
+
+<p align="center">
+  <img src="../image-driven-portrait-animation-2024-08-19.jpg" alt="LivePortrait" width="960px">
+  <br>
+  <strong>Image-driven Portrait Animation and Regional Control</strong>
+</p>
+
+### More Detailed Explanation
+
+**flag_relative_motion**:
+When using an image as the driving input, setting `--flag_relative_motion` to true will apply the motion deformation between the driving image and its canonical form. If set to false, the absolute motion of the driving image is used, which may amplify expression driving strength but could also cause identity leakage. This option corresponds to the `relative motion` toggle in the Gradio interface. Additionally, if both source and driving inputs are images, the output will be an image. If the source is a video and the driving input is an image, the output will be a video, with each frame driven by the image's motion. The Gradio interface automatically saves and displays the output in the appropriate format.
+
+**animation_region**:
+This argument offers five options:
+
+- `exp`: Only the expression of the driving input influences the source.
+- `pose`: Only the head pose drives the source.
+- `lip`: Only lip movement drives the source.
+- `eyes`: Only eye movement drives the source.
+- `all`: All motions from the driving input are applied.
+
+You can also select these options directly in the Gradio interface.
+
+**Editing the Lip Region of the Source Video to a Neutral Expression**:
+In response to requests for a more neutral lip region in the `Retargeting Video` of the Gradio interface, we've added a `keeping the lip silent` option. When selected, the animated video's lip region will adopt a neutral expression. However, this may cause inter-frame jitter or identity leakage, as it uses a mode similar to absolute driving. Note that the neutral expression may sometimes feature a slightly open mouth.
+
+**Others**:
+When both source and driving inputs are videos, the output motion may be a blend of both, due to the default setting of `--flag_relative_motion`. This option uses relative driving, where the motion offset of the current driving frame relative to the first driving frame is added to the source frame's motion. In contrast, `--no_flag_relative_motion` applies the driving frame's motion directly as the final driving motion.
+
+For CLI usage, to retain only the driving video's motion in the output, use:
+```bash
+python inference.py --no_flag_relative_motion
+```
+In the Gradio interface, simply uncheck the relative motion option. Note that absolute driving may cause jitter or identity leakage in the animated video.

ckpts/LivePortrait/assets/docs/changelog/2025-01-01.md

@@ -0,0 +1,29 @@
+## 2025/01/01
+
+**We’re thrilled that cats 🐱 are now speaking and singing across the internet!**  🎶
+
+In this update, we’ve improved the [Animals model](https://huggingface.co/KwaiVGI/LivePortrait/tree/main/liveportrait_animals/base_models_v1.1) with more data. While you might notice only a slight improvement for cats (if at all 😼), dogs have gotten a slightly better upgrade. For example, the model is now better at recognizing their mouths instead of mistaking them for noses. 🐶
+
+<table class="center" style="width: 80%; margin-left: auto; margin-right: auto;">
+<tr>
+    <td style="text-align: center"><b>Before vs. After (v1.1)</b></td>
+</tr>
+
+<tr>
+    <td style="border: none; text-align: center;">
+        <video controls loop src="https://github.com/user-attachments/assets/59fc09b9-6cb7-4265-833f-eebb27ed9511" muted="false" style="width: 60%;"></video>
+    </td>
+</tr>
+</table>
+
+
+The new version (v1.1) Animals Model has been updated on [HuggingFace](https://huggingface.co/KwaiVGI/LivePortrait/tree/main/liveportrait_animals/base_models_v1.1). The new version is enabled by default.
+
+> [!IMPORTANT]
+> Note: Make sure to update your weights to use the new version.
+
+If you prefer to use the original version, simply modify the configuration in [inference_config.py](../../../src/config/inference_config.py#L29)
+```python
+version_animals = "" # old version
+# version_animals = "_v1.1" # new (v1.1) version
+```

ckpts/LivePortrait/assets/docs/directory-structure.md

@@ -0,0 +1,28 @@
+## The directory structure of `pretrained_weights`
+
+```text
+pretrained_weights
+├── insightface
+│   └── models
+│       └── buffalo_l
+│           ├── 2d106det.onnx
+│           └── det_10g.onnx
+├── liveportrait
+│   ├── base_models
+│   │   ├── appearance_feature_extractor.pth
+│   │   ├── motion_extractor.pth
+│   │   ├── spade_generator.pth
+│   │   └── warping_module.pth
+│   ├── landmark.onnx
+│   └── retargeting_models
+│       └── stitching_retargeting_module.pth
+└── liveportrait_animals
+    ├── base_models
+    │   ├── appearance_feature_extractor.pth
+    │   ├── motion_extractor.pth
+    │   ├── spade_generator.pth
+    │   └── warping_module.pth
+    ├── retargeting_models
+    │   └── stitching_retargeting_module.pth
+    └── xpose.pth
+```

ckpts/LivePortrait/assets/docs/how-to-install-ffmpeg.md

@@ -0,0 +1,29 @@
+## Install FFmpeg
+
+Make sure you have `ffmpeg` and `ffprobe` installed on your system. If you don't have them installed, follow the instructions below.
+
+> [!Note]
+> The installation is copied from [SoVITS](https://github.com/RVC-Boss/GPT-SoVITS) 🤗
+
+### Conda Users
+
+```bash
+conda install ffmpeg
+```
+
+### Ubuntu/Debian Users
+
+```bash
+sudo apt install ffmpeg
+sudo apt install libsox-dev
+conda install -c conda-forge 'ffmpeg<7'
+```
+
+### Windows Users
+
+Download and place [ffmpeg.exe](https://huggingface.co/lj1995/VoiceConversionWebUI/blob/main/ffmpeg.exe) and [ffprobe.exe](https://huggingface.co/lj1995/VoiceConversionWebUI/blob/main/ffprobe.exe) in the GPT-SoVITS root.
+
+### MacOS Users
+```bash
+brew install ffmpeg
+```

ckpts/LivePortrait/assets/docs/speed.md

@@ -0,0 +1,13 @@
+### Speed
+
+Below are the results of inferring one frame on an RTX 4090 GPU using the native PyTorch framework with `torch.compile`:
+
+| Model                             | Parameters(M) | Model Size(MB) | Inference(ms) |
+|-----------------------------------|:-------------:|:--------------:|:-------------:|
+| Appearance Feature Extractor      |     0.84      |       3.3      |     0.82      |
+| Motion Extractor                  |     28.12     |       108      |     0.84      |
+| Spade Generator                   |     55.37     |       212      |     7.59      |
+| Warping Module                    |     45.53     |       174      |     5.21      |
+| Stitching and Retargeting Modules |     0.23      |       2.3      |     0.31      |
+
+*Note: The values for the Stitching and Retargeting Modules represent the combined parameter counts and total inference time of three sequential MLP networks.*

ckpts/LivePortrait/assets/examples/driving/d0.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63f6f9962e1fdf6e6722172e7a18155204858d5d5ce3b1e0646c150360c33bed
+size 2958395

ckpts/LivePortrait/assets/examples/driving/d10.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac7ee3c2419046f11dc230b6db33c2391a98334eba2b1d773e7eb9627992622f
+size 1064930

ckpts/LivePortrait/assets/examples/driving/d11.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94b449a25686eddd42e244fb571c908a123aa0154776682601df1f3830f8f65c
+size 468504

ckpts/LivePortrait/assets/examples/driving/d12.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2895169e5aa2a882d39dfc80a17a3eab1cf6ec23b9b6f6be76bae48deda15219
+size 596446

ckpts/LivePortrait/assets/examples/driving/d13.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d03e39c28323cde1c5fc6c5629aa83fe6c834fa7c9ed2dac969e1247eaafdb60
+size 2475854

ckpts/LivePortrait/assets/examples/driving/d14.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:465e72fbf26bf4ed46d1adf7aab8a7344aac54a2f92c4d82a1d53127f0170472
+size 891025

ckpts/LivePortrait/assets/examples/driving/d18.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1dc94c1fec7ef7dc831c8a49f0e1788ae568812cb68e62f6875d9070f573d02a
+size 187263

ckpts/LivePortrait/assets/examples/driving/d19.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3047ba66296d96b8a4584e412e61493d7bc0fa5149c77b130e7feea375e698bd
+size 232859

ckpts/LivePortrait/assets/examples/driving/d20.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e249873c21abf27b3c9f405a1b5283ef1b70e5eff21c1df44ca4b5b4d9b7309
+size 462335

ckpts/LivePortrait/assets/examples/driving/d3.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef5c86e49b1b43dcb1449b499eb5a7f0cbae2f78aec08b5598193be1e4257099
+size 1430968

ckpts/LivePortrait/assets/examples/driving/d6.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00e3ea79bbf28cbdc4fbb67ec655d9a0fe876e880ec45af55ae481348d0c0fff
+size 1967790