app.py

# -*- coding: utf-8 -*-
import gc
import os
import random
import subprocess
import sys
import uuid
from pathlib import Path

import gradio as gr
import numpy as np
import spaces
import torch
from huggingface_hub import snapshot_download
from moviepy.editor import AudioFileClip, VideoFileClip
from PIL import Image


ROOT = Path(__file__).resolve().parent
SOURCE_DIR = ROOT / "echomimic_v2"
WEIGHTS_DIR = ROOT / "pretrained_weights"
OUTPUT_DIR = ROOT / "outputs"
SOURCE_REPO = "https://github.com/antgroup/echomimic_v2.git"
WEIGHTS_REPO = "BadToBest/EchoMimicV2"
POSE_NAMES = ["01", "02", "03", "04", "fight", "good", "salute", "ultraman"]

os.environ.setdefault("GRADIO_SSR_MODE", "0")
os.environ.setdefault("GRADIO_ANALYTICS_ENABLED", "False")
os.environ.setdefault("FFMPEG_PATH", "/usr/bin")


def ensure_source():
    if not (SOURCE_DIR / "src").is_dir():
        print("[SETUP] Cloning official EchoMimicV2 source...", flush=True)
        subprocess.run(
            ["git", "clone", "--depth", "1", SOURCE_REPO, str(SOURCE_DIR)],
            check=True,
        )
    if str(SOURCE_DIR) not in sys.path:
        sys.path.insert(0, str(SOURCE_DIR))


ensure_source()

from diffusers import AutoencoderKL, DDIMScheduler
from src.models.pose_encoder import PoseEncoder
from src.models.unet_2d_condition import UNet2DConditionModel
from src.models.unet_3d_emo import EMOUNet3DConditionModel
from src.models.whisper.audio2feature import load_audio_model
from src.pipelines.pipeline_echomimicv2_acc import EchoMimicV2Pipeline
from src.utils.dwpose_util import draw_pose_select_v2
from src.utils.util import save_videos_grid


pipeline = None
weight_dtype = torch.float16
device = "cuda"


def ensure_models():
    WEIGHTS_DIR.mkdir(parents=True, exist_ok=True)
    required = [
        "denoising_unet_acc.pth",
        "motion_module_acc.pth",
        "pose_encoder.pth",
        "reference_unet.pth",
    ]
    if not all((WEIGHTS_DIR / name).is_file() for name in required):
        print("[SETUP] Downloading official EchoMimicV2 accelerated weights...", flush=True)
        snapshot_download(
            repo_id=WEIGHTS_REPO,
            local_dir=WEIGHTS_DIR,
            allow_patterns=required,
        )

    vae_dir = WEIGHTS_DIR / "sd-vae-ft-mse"
    if not (vae_dir / "config.json").is_file():
        print("[SETUP] Downloading SD VAE...", flush=True)
        snapshot_download(repo_id="stabilityai/sd-vae-ft-mse", local_dir=vae_dir)

    base_dir = WEIGHTS_DIR / "sd-image-variations-diffusers"
    if not (base_dir / "unet" / "config.json").is_file():
        print("[SETUP] Downloading SD image-variations base model...", flush=True)
        snapshot_download(
            repo_id="lambdalabs/sd-image-variations-diffusers",
            local_dir=base_dir,
        )

    whisper_path = WEIGHTS_DIR / "audio_processor" / "tiny.pt"
    if not whisper_path.is_file():
        print("[SETUP] Downloading Whisper tiny checkpoint...", flush=True)
        whisper_path.parent.mkdir(parents=True, exist_ok=True)
        torch.hub.download_url_to_file(
            "https://openaipublic.azureedge.net/main/whisper/models/"
            "65147644a518d12f04e32d6f3b26facc3f8dd46e5390956a9424a650c0ce22b9/"
            "tiny.pt",
            str(whisper_path),
        )


def log_model(name, model, source):
    first = next(model.parameters(), None) if isinstance(model, torch.nn.Module) else None
    parameters = (
        sum(parameter.numel() for parameter in model.parameters())
        if isinstance(model, torch.nn.Module)
        else None
    )
    print(
        f"[MODEL] name={name}, class={model.__class__.__name__}, source={source}, "
        f"parameters={parameters}, dtype={getattr(first, 'dtype', None)}, "
        f"device={getattr(first, 'device', None)}",
        flush=True,
    )


def load_models():
    global pipeline
    if pipeline is not None:
        return pipeline
    if not torch.cuda.is_available():
        raise RuntimeError("EchoMimicV2 Accelerated requires CUDA")

    ensure_models()
    base_dir = WEIGHTS_DIR / "sd-image-variations-diffusers"

    print("[MODEL LOAD] VAE from_pretrained", flush=True)
    vae = AutoencoderKL.from_pretrained(
        WEIGHTS_DIR / "sd-vae-ft-mse",
        torch_dtype=weight_dtype,
    ).to(device=device, dtype=weight_dtype)
    log_model("vae", vae, WEIGHTS_DIR / "sd-vae-ft-mse")

    print("[MODEL LOAD] Reference UNet from_pretrained", flush=True)
    reference_unet = UNet2DConditionModel.from_pretrained(
        base_dir,
        subfolder="unet",
        use_safetensors=False,
    ).to(device=device, dtype=weight_dtype)
    state = torch.load(WEIGHTS_DIR / "reference_unet.pth", map_location="cpu", weights_only=True)
    reference_unet.load_state_dict(state)
    del state
    log_model("reference-unet", reference_unet, WEIGHTS_DIR / "reference_unet.pth")

    print("[MODEL LOAD] Accelerated denoising UNet from_pretrained_2d", flush=True)
    unet_kwargs = {
        "use_inflated_groupnorm": True,
        "unet_use_cross_frame_attention": False,
        "unet_use_temporal_attention": False,
        "use_motion_module": True,
        "cross_attention_dim": 384,
        "motion_module_resolutions": [1, 2, 4, 8],
        "motion_module_mid_block": True,
        "motion_module_decoder_only": False,
        "motion_module_type": "Vanilla",
        "motion_module_kwargs": {
            "num_attention_heads": 8,
            "num_transformer_block": 1,
            "attention_block_types": ["Temporal_Self", "Temporal_Self"],
            "temporal_position_encoding": True,
            "temporal_position_encoding_max_len": 32,
            "temporal_attention_dim_div": 1,
        },
    }
    denoising_unet = EMOUNet3DConditionModel.from_pretrained_2d(
        str(base_dir),
        str(WEIGHTS_DIR / "motion_module_acc.pth"),
        subfolder="unet",
        unet_additional_kwargs=unet_kwargs,
    ).to(device=device, dtype=weight_dtype)
    state = torch.load(
        WEIGHTS_DIR / "denoising_unet_acc.pth",
        map_location="cpu",
        weights_only=True,
    )
    missing, unexpected = denoising_unet.load_state_dict(state, strict=False)
    del state
    print(
        f"[MODEL] accelerated-unet missing={len(missing)}, unexpected={len(unexpected)}",
        flush=True,
    )
    log_model("accelerated-denoising-unet", denoising_unet, WEIGHTS_DIR / "denoising_unet_acc.pth")

    pose_encoder = PoseEncoder(
        320,
        conditioning_channels=3,
        block_out_channels=(16, 32, 96, 256),
    ).to(device=device, dtype=weight_dtype)
    state = torch.load(WEIGHTS_DIR / "pose_encoder.pth", map_location="cpu", weights_only=True)
    pose_encoder.load_state_dict(state)
    del state
    log_model("pose-encoder", pose_encoder, WEIGHTS_DIR / "pose_encoder.pth")

    print("[MODEL LOAD] Whisper tiny audio guider", flush=True)
    audio_guider = load_audio_model(
        model_path=str(WEIGHTS_DIR / "audio_processor" / "tiny.pt"),
        device=device,
    )
    log_model("audio-guider", audio_guider, WEIGHTS_DIR / "audio_processor" / "tiny.pt")

    scheduler = DDIMScheduler(
        beta_start=0.00085,
        beta_end=0.012,
        beta_schedule="linear",
        clip_sample=False,
        steps_offset=1,
        prediction_type="v_prediction",
        rescale_betas_zero_snr=True,
        timestep_spacing="trailing",
    )
    pipeline = EchoMimicV2Pipeline(
        vae=vae,
        reference_unet=reference_unet,
        denoising_unet=denoising_unet,
        audio_guider=audio_guider,
        pose_encoder=pose_encoder,
        scheduler=scheduler,
    ).to(device, dtype=weight_dtype)
    gc.collect()
    torch.cuda.empty_cache()
    print("[MODEL LOAD COMPLETE] EchoMimicV2 Accelerated is ready", flush=True)
    return pipeline


def estimate_gpu_duration(_image, audio, *_args):
    try:
        clip = AudioFileClip(audio)
        duration = float(clip.duration)
        clip.close()
    except Exception:
        duration = 5.0
    return max(180, min(900, int(150 + duration * 35)))


def resolve_pose_dir(name):
    pose_dir = SOURCE_DIR / "assets" / "halfbody_demo" / "pose" / name
    if not pose_dir.is_dir():
        raise gr.Error(f"姿态模板不存在：{name}")
    frames = sorted(pose_dir.glob("*.npy"), key=lambda path: int(path.stem))
    if not frames:
        raise gr.Error(f"姿态模板没有可用帧：{name}")
    return pose_dir, len(frames)


@spaces.GPU(size="xlarge", duration=estimate_gpu_duration)
@torch.inference_mode()
def generate(image, audio, pose_name, max_frames, seed, progress=gr.Progress()):
    if not image:
        raise gr.Error("请上传参考图片")
    if not audio:
        raise gr.Error("请上传驱动音频")

    progress(0.05, desc="加载 Accelerated 模型...")
    pipe = load_models()
    pose_dir, pose_frames = resolve_pose_dir(pose_name)

    width = height = 768
    fps = 24
    steps = 6
    cfg = 1.0
    context_frames = 12
    context_overlap = 3
    max_frames = max(12, min(120, int(max_frames)))

    audio_clip = AudioFileClip(audio)
    audio_frames = int(audio_clip.duration * fps)
    length = min(max_frames, audio_frames, pose_frames)
    if length < 12:
        audio_clip.close()
        raise gr.Error("音频过短，至少需要约0.5秒")
    audio_clip = audio_clip.set_duration(length / fps)

    actual_seed = random.randint(1, 2**31 - 1) if seed is None or seed < 0 else int(seed)
    generator = torch.Generator(device=device).manual_seed(actual_seed)
    ref_image = Image.open(image).convert("RGB").resize((width, height))

    progress(0.15, desc="准备姿态模板...")
    pose_list = []
    for index in range(length):
        target = np.zeros((width, height, 3), dtype=np.uint8)
        detected_pose = np.load(pose_dir / f"{index}.npy", allow_pickle=True).tolist()
        imh_new, imw_new, rb, re, cb, ce = detected_pose["draw_pose_params"]
        rendered = draw_pose_select_v2(detected_pose, imh_new, imw_new, ref_w=800)
        rendered = np.transpose(np.asarray(rendered), (1, 2, 0))
        target[rb:re, cb:ce, :] = rendered
        pose_list.append(
            torch.from_numpy(target).to(device=device, dtype=weight_dtype).permute(2, 0, 1) / 255.0
        )
    poses = torch.stack(pose_list, dim=1).unsqueeze(0)

    progress(0.25, desc=f"生成 {length} 帧视频...")
    video = pipe(
        ref_image,
        audio,
        poses,
        width,
        height,
        length,
        steps,
        cfg,
        generator=generator,
        audio_sample_rate=16000,
        context_frames=context_frames,
        fps=fps,
        context_overlap=context_overlap,
        start_idx=0,
    ).videos

    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
    job_id = uuid.uuid4().hex
    silent_path = OUTPUT_DIR / f"{job_id}_silent.mp4"
    result_path = OUTPUT_DIR / f"{job_id}.mp4"
    save_videos_grid(video[:, :, :length], str(silent_path), n_rows=1, fps=fps)

    progress(0.92, desc="合并音频...")
    video_clip = VideoFileClip(str(silent_path)).set_audio(audio_clip)
    video_clip.write_videofile(
        str(result_path),
        codec="libx264",
        audio_codec="aac",
        fps=fps,
        threads=2,
        logger=None,
    )
    video_clip.close()
    audio_clip.close()
    silent_path.unlink(missing_ok=True)
    progress(1.0, desc="完成")
    return str(result_path), actual_seed, f"生成 {length} 帧（{length / fps:.1f} 秒），姿态模板：{pose_name}"


with gr.Blocks(title="EchoMimicV2 Accelerated") as demo:
    gr.Markdown(
        "# EchoMimicV2 Accelerated\n"
        "上传半身参考图和音频，使用官方加速权重与预置姿态模板生成说话视频。"
    )
    with gr.Row():
        with gr.Column():
            image_input = gr.Image(label="半身参考图片", type="filepath", sources=["upload"])
            audio_input = gr.Audio(label="驱动音频", type="filepath", sources=["upload"])
            pose_input = gr.Dropdown(
                choices=POSE_NAMES,
                value="01",
                label="动作/姿态模板",
            )
            max_frames_input = gr.Slider(
                minimum=24,
                maximum=120,
                value=72,
                step=12,
                label="最大生成帧数（24fps）",
            )
            seed_input = gr.Number(value=-1, precision=0, label="随机种子（-1为随机）")
            generate_button = gr.Button("生成视频", variant="primary")
        with gr.Column():
            video_output = gr.Video(label="生成结果", autoplay=False)
            seed_output = gr.Number(label="实际种子", precision=0)
            info_output = gr.Textbox(label="生成信息")

    gr.Markdown(
        "首次运行需下载约17GB模型。官方加速配置为768×768、6步；"
        "姿态模板决定身体动作，音频主要控制口型和节奏。"
    )
    generate_button.click(
        generate,
        inputs=[image_input, audio_input, pose_input, max_frames_input, seed_input],
        outputs=[video_output, seed_output, info_output],
    )

demo.queue(default_concurrency_limit=1, max_size=6)

if __name__ == "__main__":
    demo.launch()