Upload folder using huggingface_hub

README.md

@@ -1,13 +1,42 @@
 ---
-title: Prism Text To Motion
-emoji: 📉
-colorFrom: red
-colorTo: indigo
+title: PRISM Text-to-Motion
+emoji: 🏃
+colorFrom: purple
+colorTo: yellow
 sdk: gradio
-sdk_version: 6.24.0
-python_version: '3.12'
+sdk_version: 5.50.0
 app_file: app.py
-pinned: false
+short_description: Generate human motion sequences from text prompts
+python_version: "3.12"
+startup_duration_timeout: 30m
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# PRISM: Streaming Human Motion Generation with Per-Joint Latent Decomposition
+
+This Space demonstrates **PRISM**, a text-to-motion generation model that produces
+SMPL body motion sequences from natural language prompts.
+
+## How it works
+
+1. Enter a text prompt describing a human motion
+2. The model generates a motion sequence using a flow-matching DiT transformer
+   with a causal spatio-temporal Motion VAE
+3. The output is rendered as a 3D skeleton animation
+
+## Model
+
+- **Model**: `ZeyuLing/PRISM-TP2M-1.4B` (~1.4B parameters)
+- **Architecture**: Flow-matching DiT transformer with per-joint latent decomposition
+- **Text encoder**: UMT5 (T5-style)
+- **Output**: SMPL body parameters (22 joints, rotation_6d, 30 fps)
+
+## Citation
+
+```bibtex
+@article{ling2026prism,
+  title={PRISM: Streaming Human Motion Generation with Per-Joint Latent Decomposition},
+  author={Ling, Zeyu and Shuai, Qing and Zhang, Teng and Li, Shiyang and Han, Bo and Zou, Changqing},
+  journal={arXiv preprint arXiv:2603.08590},
+  year={2026}
+}
+```

app.py

@@ -0,0 +1,365 @@
+import spaces  # MUST be first
+import torch
+import numpy as np
+import os
+import sys
+import tempfile
+import subprocess
+import json
+import logging
+
+# ── Clone PRISM repo at module scope ──────────────────────────────────────────
+PRISM_REPO_DIR = "/tmp/prism_src"
+if not os.path.isdir(PRISM_REPO_DIR):
+    subprocess.run(
+        ["git", "clone", "--depth", "1", "https://github.com/ZeyuLing/PRISM.git", PRISM_REPO_DIR],
+        check=True,
+        capture_output=True,
+    )
+sys.path.insert(0, PRISM_REPO_DIR)
+
+# ── Download auxiliary files at module scope ────────────────────────────────
+from huggingface_hub import hf_hub_download, snapshot_download
+
+# Model weights
+MODEL_ID = "ZeyuLing/PRISM-TP2M-1.4B"
+model_dir = snapshot_download(MODEL_ID)
+
+# Tokenizer from the motius-prism repo (same architecture, shares UMT5 tokenizer)
+tokenizer_dir = snapshot_download(
+    "ZeyuLing/motius-prism-1.0-humanml3d",
+    allow_patterns=["tokenizer/*"],
+)
+tokenizer_path = os.path.join(tokenizer_dir, "tokenizer")
+
+# Stats file
+stats_file = hf_hub_download(
+    "ZeyuLing/motius-prism-1.0-humanml3d", "motion_stats.json"
+)
+
+# SMPL model files
+SMPL_DIR = "/tmp/smpl_models/smplx"
+os.makedirs(SMPL_DIR, exist_ok=True)
+
+# SMPLX_NEUTRAL.npz from gvhmr_ckp dataset
+smplx_neutral_path = hf_hub_download(
+    "wendell0218/gvhmr_ckp",
+    "body_models/smplx/SMPLX_NEUTRAL.npz",
+    repo_type="dataset",
+    local_dir=SMPL_DIR,
+)
+
+# SMPL helper files from GVHMR GitHub repo
+import urllib.request
+GVHMR_RAW = "https://raw.githubusercontent.com/zju3dv/GVHMR/main/hmr4d/utils/body_model"
+for fname in ["smplx2smpl_sparse.pt", "smpl_coco17_J_regressor.pt", "smplx_verts437.pt"]:
+    fpath = os.path.join(SMPL_DIR, fname)
+    if not os.path.isfile(fpath):
+        urllib.request.urlretrieve(f"{GVHMR_RAW}/{fname}", fpath)
+
+# ── Set environment variables for the pipeline ──────────────────────────────
+os.environ["PRISM_TOKENIZER_PATH"] = tokenizer_path
+os.environ["PRISM_STATS_FILE"] = stats_file
+os.environ["PRISM_SMPL_MODEL_PATH"] = SMPL_DIR
+
+# ── Load the pipeline ────────────────────────────────────────────────────────
+from prism.pipelines.prism_from_pretrained import load_prism_pipeline_from_pretrained
+
+pipe = load_prism_pipeline_from_pretrained(
+    model_dir,
+    device="cuda",
+    torch_dtype=torch.bfloat16,
+)
+pipe.transformer.eval()
+
+# ── SMPL skeleton for rendering ──────────────────────────────────────────────
+# SMPL 22-joint skeleton connections (parent→child)
+SMPL_SKELETON = [
+    (0, 1), (0, 2), (0, 3), (1, 4), (2, 5), (3, 6),
+    (4, 7), (5, 8), (6, 9), (7, 10), (8, 11),
+    (0, 12), (0, 13), (12, 14), (13, 15), (14, 16),
+    (15, 17), (16, 18), (17, 19), (18, 20),
+    (9, 21),  # right hand → right wrist (using 22nd joint if available)
+]
+
+# Standard SMPL parent indices
+SMPL_PARENTS = [
+    -1, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 0, 12, 13, 14, 15, 16, 17, 18, 9
+]
+
+# Joint connections for SMPL 22-joint model
+SMPL_JOINT_PAIRS = [
+    (0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6),   # right arm
+    (0, 7), (7, 8), (8, 9), (9, 10), (10, 11),         # left arm (reversed)
+    (0, 12), (12, 13), (13, 14), (14, 15),             # right leg
+    (0, 16), (16, 17), (17, 18), (18, 19),             # left leg
+    (3, 20),  # neck→head
+    (9, 21),  # left wrist→left hand
+]
+
+# Joint names for labels
+JOINT_NAMES = [
+    "root", "r_hip", "r_knee", "r_ankle", "r_foot",
+    "l_hip", "l_knee", "l_ankle", "l_foot",
+    "spine", "neck", "head",
+    "l_collar", "l_shoulder", "l_elbow", "l_wrist", "l_hand",
+    "r_collar", "r_shoulder", "r_elbow", "r_wrist", "r_hand"
+]
+
+
+def smplx_dict_to_joints(smplx_dict):
+    """Convert SMPL-X output dict to 3D joint positions for rendering.
+
+    Uses the SmplxLite FK model to compute 3D joint positions from
+    the axis-angle rotations and translation.
+
+    Returns:
+        joints: (T, 22, 3) numpy array of 3D joint positions
+    """
+    from prism.models.body_models.smplx_lite import SmplxLite
+
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    # Load the SMPL model (same path as pipeline)
+    smpl_model = SmplxLite(
+        model_path=SMPL_DIR,
+        gender="neutral",
+        num_betas=10,
+    ).to(device=device, dtype=torch.float32)
+    smpl_model.eval()
+
+    transl = torch.from_numpy(smplx_dict["transl"]).float().to(device)  # (T, 3)
+    global_orient = torch.from_numpy(smplx_dict["global_orient"]).float().to(device)  # (T, 3)
+    body_pose = torch.from_numpy(smplx_dict["body_pose"]).float().to(device)  # (T, 63)
+
+    T = transl.shape[0]
+    # Process in chunks to avoid OOM
+    chunk = 128
+    all_joints = []
+    with torch.no_grad():
+        for i in range(0, T, chunk):
+            t = transl[i:i+chunk]  # (C, 3)
+            go = global_orient[i:i+chunk]  # (C, 3)
+            bp = body_pose[i:i+chunk]  # (C, 63)
+            C = t.shape[0]
+            joints, _, _ = smpl_model.fk(
+                transl=t.unsqueeze(0),           # (1, C, 3)
+                global_orient=go.unsqueeze(0),   # (1, C, 3)
+                body_pose=bp.unsqueeze(0),      # (1, C, 63)
+                betas=torch.zeros(1, C, 10, device=device, dtype=torch.float32),
+            )
+            # joints: (1, C, 22, 3)
+            all_joints.append(joints.squeeze(0).cpu().numpy())
+
+    joints = np.concatenate(all_joints, axis=0)  # (T, 22, 3)
+    return joints
+
+
+def render_skeleton_video(joints, fps=30, figsize=(6, 6), dpi=100):
+    """Render a (T, 22, 3) joint array to an MP4 video using matplotlib.
+
+    Returns path to the temporary video file.
+    """
+    import matplotlib
+    matplotlib.use("Agg")
+    import matplotlib.pyplot as plt
+    from mpl_toolkits.mplot3d import Axes3D
+    import imageio
+
+    T, J, _ = joints.shape
+
+    # Compute global coordinate range for consistent view
+    all_pts = joints.reshape(-1, 3)
+    mins = all_pts.min(axis=0)
+    maxs = all_pts.max(axis=0)
+    center = (mins + maxs) / 2
+    extent = (maxs - mins).max() / 2 * 1.2
+
+    # Create temporary directory for frames
+    tmpdir = tempfile.mkdtemp()
+    frame_paths = []
+
+    for t in range(T):
+        fig = plt.figure(figsize=figsize, dpi=dpi)
+        ax = fig.add_subplot(111, projection="3d")
+
+        pts = joints[t]  # (22, 3)
+
+        # Plot skeleton lines
+        for (i, j) in SMPL_JOINT_PAIRS:
+            if i < J and j < J:
+                ax.plot(
+                    [pts[i, 0], pts[j, 0]],
+                    [pts[i, 1], pts[j, 1]],
+                    [pts[i, 2], pts[j, 2]],
+                    color="steelblue",
+                    linewidth=2.5,
+                )
+
+        # Plot joints
+        ax.scatter(pts[:, 0], pts[:, 1], pts[:, 2], c="firebrick", s=30, zorder=5)
+
+        # Set consistent view
+        ax.set_xlim(center[0] - extent, center[0] + extent)
+        ax.set_ylim(center[1] - extent, center[1] + extent)
+        ax.set_zlim(center[2] - extent, center[2] + extent)
+        ax.set_xlabel("X")
+        ax.set_ylabel("Z")
+        ax.set_zlabel("Y")
+        ax.set_title(f"Frame {t+1}/{T}", fontsize=10)
+        ax.view_init(elev=15, azim=-60)
+
+        # Elev=15, looking from a good angle
+        ax.set_box_aspect([1, 1, 1])
+
+        fpath = os.path.join(tmpdir, f"frame_{t:04d}.png")
+        fig.savefig(fpath, bbox_inches="tight", pad_inches=0.1)
+        plt.close(fig)
+        frame_paths.append(fpath)
+
+    # Create video
+    video_path = os.path.join(tmpdir, "motion.mp4")
+    writer = imageio.get_writer(video_path, fps=fps, codec="libx264")
+    for fpath in frame_paths:
+        writer.append_data(imageio.imread(fpath))
+    writer.close()
+
+    # Clean up frame images (keep video)
+    for fpath in frame_paths:
+        os.remove(fpath)
+
+    return video_path
+
+
+@spaces.GPU(duration=120)
+def generate_motion(
+    prompt: str,
+    num_frames: int = 129,
+    guidance_scale: float = 5.0,
+    num_inference_steps: int = 50,
+    seed: int = 0,
+):
+    """Generate a human motion sequence from a text prompt using PRISM.
+
+    Args:
+        prompt: Text description of the motion to generate.
+        num_frames: Number of motion frames (higher = longer motion, ~4s at 30fps).
+        guidance_scale: Classifier-free guidance scale (higher = more text-adherent).
+        num_inference_steps: Number of denoising steps (higher = better quality).
+        seed: Random seed for reproducibility.
+    """
+    # Set seed
+    torch.manual_seed(seed)
+    if torch.cuda.is_available():
+        torch.cuda.manual_seed(seed)
+
+    # Generate motion
+    smplx_dict = pipe(
+        prompts=prompt,
+        negative_prompt="",
+        num_frames_per_segment=num_frames,
+        num_joints=23,
+        guidance_scale=guidance_scale,
+        num_inference_steps=num_inference_steps,
+    )
+
+    # Convert to 3D joint positions
+    joints = smplx_dict_to_joints(smplx_dict)
+
+    # Render to video
+    video_path = render_skeleton_video(joints, fps=30)
+
+    # Also save the motion data as npz
+    output_npz = os.path.join(tempfile.gettempdir(), "motion_output.npz")
+    np.savez(output_npz, **smplx_dict)
+
+    num_frames_actual = smplx_dict["transl"].shape[0]
+    duration_sec = num_frames_actual / 30.0
+    info = f"Generated {num_frames_actual} frames ({duration_sec:.1f}s) at 30 FPS"
+
+    return video_path, output_npz, info
+
+
+import gradio as gr
+
+with gr.Blocks(theme=gr.themes.Citrus()) as demo:
+    gr.Markdown(
+        """
+        # 🏃 PRISM: Text-to-Motion Generation
+
+        Generate human motion sequences from text prompts using
+        [PRISM](https://github.com/ZeyuLing/PRISM) — a flow-matching DiT transformer
+        with per-joint latent decomposition for streaming motion synthesis.
+
+        The model outputs SMPL body parameters (22 joints) rendered as a 3D skeleton animation.
+        """
+    )
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            prompt_input = gr.Textbox(
+                label="Motion Description",
+                placeholder="e.g. A person walks forward and waves their hand.",
+                lines=3,
+                value="A person walks forward and waves their hand.",
+            )
+            generate_btn = gr.Button("Generate Motion", variant="primary", size="lg")
+
+            with gr.Accordion("Advanced Settings", open=False):
+                num_frames_slider = gr.Slider(
+                    label="Number of Frames",
+                    minimum=33,
+                    maximum=257,
+                    step=4,
+                    value=129,
+                    info="Higher = longer motion (~4.3s at 30fps for 129 frames)",
+                )
+                guidance_slider = gr.Slider(
+                    label="Guidance Scale",
+                    minimum=1.0,
+                    maximum=15.0,
+                    step=0.5,
+                    value=5.0,
+                    info="Higher = more text-adherent, lower = more diverse",
+                )
+                steps_slider = gr.Slider(
+                    label="Inference Steps",
+                    minimum=10,
+                    maximum=100,
+                    step=5,
+                    value=50,
+                    info="Higher = better quality, slower",
+                )
+                seed_input = gr.Number(
+                    label="Seed",
+                    value=0,
+                    precision=0,
+                )
+
+        with gr.Column(scale=1):
+            video_output = gr.Video(label="Generated Motion", format="mp4")
+            info_output = gr.Textbox(label="Info", interactive=False)
+            npz_output = gr.File(label="Motion Data (.npz)")
+
+    gr.Examples(
+        examples=[
+            ["A person walks forward and waves their hand.", 129, 5.0, 50, 0],
+            ["A person performs a backflip.", 129, 7.0, 50, 42],
+            ["A person sits down on the floor cross-legged.", 129, 5.0, 50, 7],
+            ["A person dances happily, spinning around.", 193, 6.0, 50, 123],
+            ["A person kicks a soccer ball.", 97, 5.0, 50, 99],
+        ],
+        inputs=[prompt_input, num_frames_slider, guidance_slider, steps_slider, seed_input],
+        fn=generate_motion,
+        outputs=[video_output, npz_output, info_output],
+        cache_examples=True,
+        cache_mode="lazy",
+    )
+
+    generate_btn.click(
+        fn=generate_motion,
+        inputs=[prompt_input, num_frames_slider, guidance_slider, steps_slider, seed_input],
+        outputs=[video_output, npz_output, info_output],
+    )
+
+demo.launch(mcp_server=True)

requirements.txt

@@ -0,0 +1,12 @@
+# PRISM dependencies
+# Do NOT list gradio, spaces, or huggingface_hub (preinstalled)
+diffusers>=0.32.0
+transformers>=4.45.0
+accelerate
+mmengine
+einops
+smplx
+matplotlib
+imageio
+imageio-ffmpeg
+safetensors