app.py

"""HandX bimanual text-to-motion — Hugging Face Space demo.

Generates two-hand motion from text (left hand / right hand / interaction) using the
released HandX diffusion checkpoints, and renders a skeleton animation. MANO is not used.
"""
import os
import tempfile

import numpy as np
import torch
import gradio as gr

# Workaround for a gradio_client bug: it crashes parsing JSON schemas where a value is a
# bare bool (e.g. additionalProperties: true). Patch it to degrade gracefully.
import gradio_client.utils as _gcu
_orig_j2p = _gcu._json_schema_to_python_type
def _safe_j2p(schema, defs=None):
    if isinstance(schema, bool):
        return "Any"
    return _orig_j2p(schema, defs)
_gcu._json_schema_to_python_type = _safe_j2p

from einops import rearrange
from omegaconf import OmegaConf
from huggingface_hub import hf_hub_download

# ZeroGPU support: no-op decorator when running outside HF Spaces.
try:
    import spaces
    GPU = spaces.GPU(duration=120)
except Exception:  # noqa: BLE001
    def GPU(fn):
        return fn

import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation

from src.diffusion.utils.model_utils import create_model_and_diffusion
from src.diffusion.model.cls_free_sampler import ClassifierFreeSampleWrapper
from src.visualize.skeleton_visualizer import Skeleton_Visualize_Helper

MODEL_REPO = "alexzhang598/HandX-diffusion"
VARIANTS = {"layers12 (best)": "layers12", "layers8": "layers8", "layers4": "layers4"}
MOTION_LENGTH = 60
GUIDANCE = 2.5
REPR_DIMS = {"joint_pos": (42, 3), "joint_pos_w_scalar_rot": (42, 4), "joint_rot": (34, 6)}

_MEAN = np.load(os.path.join(os.path.dirname(__file__), "mean_std", "mean.npy"))
_STD = np.load(os.path.join(os.path.dirname(__file__), "mean_std", "std.npy"))

_cache = {}  # variant -> (model, diffusion, njoints, nfeats)


def _load(variant: str):
    if variant in _cache:
        return _cache[variant]
    cfg = OmegaConf.load(hf_hub_download(MODEL_REPO, f"{variant}/config.yaml"))
    njoints, nfeats = REPR_DIMS[cfg.data.repr]
    model, diffusion = create_model_and_diffusion(cfg.model)
    sd = torch.load(hf_hub_download(MODEL_REPO, f"{variant}/model.pt"), map_location="cpu")["state_dict"]
    model.load_state_dict(sd, strict=False)  # missing keys = frozen T5 (from t5-base)
    model = ClassifierFreeSampleWrapper(model, scale=GUIDANCE).eval()
    _cache[variant] = (model, diffusion, njoints, nfeats)
    return _cache[variant]


@GPU
def generate(variant_label, left_text, right_text, relation_text, seed, guidance):
    variant = VARIANTS[variant_label]
    model, diffusion, njoints, nfeats = _load(variant)

    device = "cuda" if torch.cuda.is_available() else "cpu"
    model = model.to(device)
    model.scale = float(guidance)  # classifier-free guidance strength
    seed = int(seed)
    if seed < 0:
        seed = int(torch.randint(0, 2**31 - 1, (1,)).item())
    torch.manual_seed(seed)

    shape = (1, njoints, nfeats, MOTION_LENGTH)
    model_kwargs = dict(y=dict(
        lengths=torch.as_tensor([MOTION_LENGTH], device=device),
        text=dict(left=[left_text or ""], right=[right_text or ""],
                  two_hands_relation=[relation_text or ""]),
    ))
    with torch.no_grad():
        samples = diffusion.p_sample_loop(model, shape, clip_denoised=False,
                                          model_kwargs=model_kwargs, device=device, progress=False)
    samples = rearrange(samples, "b j f t -> b t (j f)").cpu().numpy()
    samples = samples * _STD.reshape(-1) + _MEAN.reshape(-1)  # inv_transform

    motion = samples[0].reshape(MOTION_LENGTH, njoints, nfeats)[:, :, :3]  # (T, 42, 3)
    left_motion, right_motion = motion[:, :21], motion[:, 21:]

    out_path = os.path.join(tempfile.mkdtemp(), "handx.mp4")
    render(left_motion, right_motion, out_path)
    return out_path, f"🎉 Done · seed={seed} · guidance={guidance:g} · {variant}"


def render(left_motion, right_motion, out_path, fps=30):
    """Render a single bimanual skeleton animation as a browser-playable H.264 mp4."""
    fig = plt.figure(figsize=(7, 7))
    ax = fig.add_subplot(111, projection="3d")
    helper = Skeleton_Visualize_Helper(ax, left_motion, right_motion)
    helper.initialize_ax()
    ax.view_init(elev=15, azim=-70)
    fig.tight_layout()

    frames = left_motion.shape[0]

    def update(frame):
        helper.draw(frame)

    ani = FuncAnimation(fig, update, frames=frames)
    # H.264 + yuv420p + faststart so the video plays inline in browsers (moov atom at front),
    # not just downloadable.
    ani.save(out_path, writer="ffmpeg", fps=fps,
             extra_args=["-vcodec", "libx264", "-pix_fmt", "yuv420p", "-movflags", "+faststart"])
    plt.close(fig)


EXAMPLES = [
    ["layers12 (best)",
     "The four fingers slowly straighten out from a bent position and then begin to bend again.",
     "The fingers mirror the left hand's motion, opening and then closing quickly. As the hand closes, the ring finger's middle joint hyperextends.",
     "The hands move in a synchronized manner while remaining far apart and never interacting.", 42, GUIDANCE],
    ["layers12 (best)",
     "The hand holds a static pose, with the only motion being the index and middle fingers closing together at the start.",
     "The hand is mostly still, but the pinky finger quickly bends twice mid-sequence.",
     "The fingertips of the index and middle fingers of both hands repeatedly make contact with each other throughout the motion.", 42, GUIDANCE],
]

APP_CSS = """
.gradio-container {max-width: 1200px !important; margin: auto;}
#title h1 {margin-bottom: 2px;}
#gen-btn {font-size: 1.05rem;}
#video-out video {border-radius: 12px; box-shadow: 0 4px 20px rgba(0,0,0,0.12); background:#000;}
footer {visibility: hidden; height: 0;}
"""

with gr.Blocks(title="HandX Text-to-Motion", theme=gr.themes.Soft(), css=APP_CSS) as demo:
    gr.Markdown(
        "# 🤲 HandX — Bimanual Text-to-Motion\n"
        "Generate two-hand motion from text. Describe the **left hand**, the **right hand**, "
        "and their **interaction**. "
        "[Paper](https://arxiv.org/abs/2603.28766) · "
        "[Dataset](https://huggingface.co/datasets/alexzhang598/HandX) · "
        "[Models](https://huggingface.co/alexzhang598/HandX-diffusion)",
        elem_id="title",
    )
    with gr.Row(equal_height=False):
        with gr.Column(scale=2):
            variant = gr.Dropdown(list(VARIANTS.keys()), value="layers12 (best)", label="Model")
            left = gr.Textbox(label="Left hand", lines=2,
                              placeholder="e.g. The fingers slowly straighten from a bent position.")
            right = gr.Textbox(label="Right hand", lines=2,
                               placeholder="e.g. The fingers mirror the left hand, opening then closing.")
            relation = gr.Textbox(label="Two-hand interaction", lines=2,
                                  placeholder="e.g. The hands move in a synchronized manner.")
            with gr.Accordion("Advanced settings", open=False):
                with gr.Row():
                    seed = gr.Number(value=42, label="Seed (-1 = random)", precision=0, scale=3)
                    rand_btn = gr.Button("🎲", scale=1, min_width=50)
                guidance = gr.Slider(1.0, 10.0, value=GUIDANCE, step=0.1,
                                     label="Guidance scale (CFG)",
                                     info="Higher = follow text more closely")
            btn = gr.Button("Generate motion", variant="primary", elem_id="gen-btn")
            status = gr.Markdown("")
        with gr.Column(scale=3):
            out = gr.Video(label="Generated motion", autoplay=True, loop=True, elem_id="video-out")

    inputs = [variant, left, right, relation, seed, guidance]
    rand_btn.click(lambda: int(torch.randint(0, 2**31 - 1, (1,)).item()), outputs=seed)
    btn.click(lambda: "⏳ Generating motion, please wait…", outputs=status) \
       .then(generate, inputs, [out, status])
    gr.Examples(EXAMPLES, inputs, cache_examples=False)

if __name__ == "__main__":
    demo.launch(show_api=False)