import os
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
os.environ.setdefault("MPLBACKEND", "Agg")
import re
import types
import tempfile
import spaces
import torch
import numpy as np
import gradio as gr
import matplotlib
matplotlib.use("Agg")
from huggingface_hub import hf_hub_download
from transformers import AutoModelForCausalLM, AutoTokenizer
from motionvq.vqvae import HumanVQVAE
from motionvq.motion_utils import recover_from_ric, plot_3d_motion
from motionvq.paramUtil import t2m_kinematic_chain
# ----------------------------------------------------------------------------
# Config -- values verified against the IRG-MotionLLM / Motion-Agent source and
# the actual Stage-3 checkpoint tensor shapes:
# * base LLM : google/gemma-2-2b-it (bf16)
# * added toks : , , and 512 motion codes (nb_code = 512)
# * VQVAE : HumanML3D 263-dim, codebook 512x512, down_t=2, stride_t=2
# * Stage-3 checkpoint bundles BOTH the merged full LLM and the VQVAE weights
# ----------------------------------------------------------------------------
BASE_LLM = "google/gemma-2-2b-it"
CKPT_REPO = "Lymann/IRG-MotionLLM-HumanML3D"
CKPT_FILE = "Stage-3/irg_motionllm_unified_rl_stage3.bin"
NB_CODE = 512
JOINTS_NUM = 22
FPS = 20
HF_TOKEN = os.environ.get("HF_TOKEN")
class VQArgs:
"""Minimal args namespace expected by HumanVQVAE / VQVAE_251."""
dataname = "t2m"
quantizer = "ema_reset"
mu = 0.99
VQ_ARGS = VQArgs()
# System prompt for the IRG (interleaved generation-assessment-refinement) task,
# copied verbatim from the repo's TextProcessor (unified_mogen_cot_v3).
IRG_SYSTEM_PROMPT = (
"You are an assistant who helps users understand or generate 3D human "
"motion representations."
)
IRG_USER_TEMPLATE = (
"Given a text outlining a human motion objective, employ a step-by-step "
"thought process to realize the motion: (1) analyze the text, providing a "
"clear explanation of the reasoning to identify essential elements; "
"(2) conduct several rounds of motion generation and self-assessment until "
"the motion is satisfactory. Wrap all responses in and "
"tags, and formulate a plan before each step.\nGoal Text: {caption}"
)
# ----------------------------------------------------------------------------
# Load model + VQVAE at module scope (ZeroGPU: .to("cuda") is intercepted)
# ----------------------------------------------------------------------------
print("Loading tokenizer + base LLM ...")
tokenizer = AutoTokenizer.from_pretrained(BASE_LLM, token=HF_TOKEN)
llm = AutoModelForCausalLM.from_pretrained(
BASE_LLM,
torch_dtype=torch.bfloat16,
attn_implementation="sdpa",
token=HF_TOKEN,
)
# Recreate the exact tokenizer vocabulary the model was trained with:
# base vocab + + + ..
NB_TEXT_TOKENS = len(tokenizer)
tokenizer.add_tokens(["", ""])
for i in range(NB_CODE):
tokenizer.add_tokens([f""])
llm.resize_token_embeddings(len(tokenizer))
print("Loading Stage-3 checkpoint ...")
ckpt_path = hf_hub_download(CKPT_REPO, CKPT_FILE, token=HF_TOKEN)
state = torch.load(ckpt_path, map_location="cpu")
if isinstance(state, dict) and "state_dict" in state:
state = state["state_dict"]
# The Stage-3 checkpoint is a full fine-tune (LoRA already merged) and stores
# both the LLM weights (prefixed "llm.") and the VQVAE weights (prefixed "net.").
llm_sd = {}
net_sd = {}
for k, v in state.items():
if k.startswith("llm."):
llm_sd[k[len("llm."):]] = v
elif k.startswith("net."):
net_sd[k[len("net."):]] = v
missing, unexpected = llm.load_state_dict(llm_sd, strict=False)
print(f"LLM load -> missing={len(missing)} unexpected={len(unexpected)}")
llm = llm.to(torch.bfloat16).eval()
print("Building + loading VQVAE ...")
net = HumanVQVAE(
VQ_ARGS,
nb_code=NB_CODE,
code_dim=512,
output_emb_width=512,
down_t=2,
stride_t=2,
width=512,
depth=3,
dilation_growth_rate=3,
activation="relu",
norm=None,
)
vq_missing, vq_unexpected = net.load_state_dict(net_sd, strict=False)
print(f"VQVAE load -> missing={len(vq_missing)} unexpected={len(vq_unexpected)}")
net = net.eval().float()
for p in net.parameters():
p.requires_grad = False
# HumanML3D normalization statistics (263-dim), extracted from the same
# T2M-GPT extractor bundle the authors use.
MEAN = np.load(os.path.join(os.path.dirname(__file__), "mean.npy"))
STD = np.load(os.path.join(os.path.dirname(__file__), "std.npy"))
llm.to("cuda")
net.to("cuda")
EOS_ID = tokenizer.eos_token_id
MOTION_ID_START = len(tokenizer) - (NB_CODE + 2) # first token id
# ----------------------------------------------------------------------------
# Motion-token extraction helpers (ported from mllm_single_lora.py)
# ----------------------------------------------------------------------------
def _find_seq(tokens_list, target):
for i in range(len(tokens_list) - len(target) + 1):
if tokens_list[i:i + len(target)] == target:
return i
return -1
def _extract_answer_scores(scores, ids_list):
"""Return the score rows and matching ids inside ..,
else fall back to the last .. span, else all."""
ans_start = tokenizer.encode("", add_special_tokens=False)
ans_end = tokenizer.encode("", add_special_tokens=False)
s = _find_seq(ids_list, ans_start)
if s != -1:
s2 = s + len(ans_start)
e = _find_seq(ids_list[s2:], ans_end)
if e != -1:
e += s2
return scores[s2:e], ids_list[s2:e]
return scores, ids_list
def _tokens_from_scores_and_ids(scores, ids_list):
"""Prefer already-decoded motion token ids that fall in the motion range;
otherwise argmax over the motion-logit slice. Mirrors the repo logic."""
motion_logits = scores[:, -(NB_CODE + 2):]
argmax_ids = torch.argmax(motion_logits, dim=-1)
out = []
for i, tid in enumerate(ids_list):
if MOTION_ID_START <= tid < len(tokenizer):
out.append(tid - MOTION_ID_START)
else:
out.append(int(argmax_ids[i].item()))
motion_tokens = torch.tensor(out, dtype=torch.long)
if 1 in motion_tokens.tolist():
motion_tokens = motion_tokens[:motion_tokens.tolist().index(1)]
if 0 in motion_tokens.tolist():
motion_tokens = motion_tokens[motion_tokens.tolist().index(0) + 1:]
motion_tokens = torch.clamp(motion_tokens - 2, min=0)
return motion_tokens
def _extract_last_motion_span_ids(ids_list):
st_id = tokenizer.encode("", add_special_tokens=False)[0]
ed_id = tokenizer.encode("", add_special_tokens=False)[0]
starts = [i for i, t in enumerate(ids_list) if t == st_id]
ends = [i for i, t in enumerate(ids_list) if t == ed_id]
if not starts or not ends:
return None
# last valid ... pair
for st in reversed(starts):
later_ends = [e for e in ends if e > st]
if later_ends:
return st, later_ends[0]
return None
def build_prompt(caption):
caption = (caption or "").strip()
user_prompt = IRG_USER_TEMPLATE.format(caption=caption)
text = IRG_SYSTEM_PROMPT + "\n\n" + "User: " + user_prompt + "\n\n" + " Response:"
return text
def decode_motion_to_joints(motion_tokens):
"""VQVAE token ids -> 263-dim features -> denormalize -> 22-joint xyz."""
motion_tokens = motion_tokens.to("cuda").long()
feats = net.forward_decoder(motion_tokens) # (1, T, 263)
feats = feats.detach().cpu().numpy()[0]
feats = MEAN + feats * STD # denormalize
joints = recover_from_ric(torch.from_numpy(feats).float(), JOINTS_NUM)
return joints.numpy() # (T, 22, 3)
def render_video(joints, title):
out_path = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name
plot_3d_motion(out_path, t2m_kinematic_chain, joints, title=title, fps=FPS, radius=4)
return out_path
# ----------------------------------------------------------------------------
# Inference
# ----------------------------------------------------------------------------
def _duration(caption, reasoning, max_new_tokens, *a, **k):
base = 55
if reasoning:
base = 110
return min(160, base + int(int(max_new_tokens) / 20))
@spaces.GPU(duration=_duration)
def generate(caption, reasoning=True, max_new_tokens=1300, seed=0,
progress=gr.Progress(track_tqdm=True)):
if not caption or not caption.strip():
raise gr.Error("Please enter a motion description.")
if seed and int(seed) > 0:
torch.manual_seed(int(seed))
do_sample = bool(seed and int(seed) > 0)
max_new_tokens = int(max_new_tokens)
prompt = build_prompt(caption)
enc = tokenizer(prompt, return_tensors="pt").to("cuda")
input_len = enc.input_ids.shape[1]
if reasoning:
max_len = input_len + max_new_tokens
else:
# skip the interleaved reasoning; ask for a compact motion answer
max_len = input_len + 260
gen_kwargs = dict(
max_length=max_len,
do_sample=do_sample,
return_dict_in_generate=True,
output_scores=True,
use_cache=True,
)
if do_sample:
gen_kwargs["temperature"] = 1.0
with torch.inference_mode():
outputs = llm.generate(enc.input_ids, attention_mask=enc.attention_mask, **gen_kwargs)
gen_ids = outputs.sequences[0, input_len:]
# scores are per generated step (tuple len == n_new); stack into (n_new, vocab)
scores = torch.stack(outputs.scores)[:, 0, :]
# truncate at EOS
gen_ids_list = gen_ids.tolist()
if EOS_ID in gen_ids_list:
cut = gen_ids_list.index(EOS_ID) + 1
gen_ids_list = gen_ids_list[:cut]
scores = scores[:cut]
full_text = tokenizer.decode(gen_ids_list, skip_special_tokens=False)
# Prefer ..; fall back to last .. span.
ans_scores, ans_ids = _extract_answer_scores(scores, gen_ids_list)
if ans_ids is gen_ids_list or len(ans_ids) == len(gen_ids_list):
span = _extract_last_motion_span_ids(gen_ids_list)
if span is not None:
st, ed = span
ans_scores = scores[st + 1:ed]
ans_ids = gen_ids_list[st + 1:ed]
motion_tokens = _tokens_from_scores_and_ids(ans_scores, ans_ids)
if motion_tokens.numel() == 0:
raise gr.Error(
"The model did not produce a valid motion for this prompt. "
"Try rephrasing, or toggle the reasoning option."
)
joints = decode_motion_to_joints(motion_tokens)
title = caption.strip()
if len(title) > 60:
title = title[:57] + "..."
video = render_video(joints, title)
# Build a readable reasoning trace (strip the internal [plan]/[tag] markup).
trace = full_text
trace = trace.replace("", "").replace("", "")
n_rounds = len(re.findall(r"\[generate\]", trace))
info = (
f"Frames: {joints.shape[0]} | Motion tokens: {motion_tokens.numel()}"
f" | Internal generate/refine rounds: {max(n_rounds, 1)}"
)
return video, info, trace.strip()
# ----------------------------------------------------------------------------
# UI
# ----------------------------------------------------------------------------
DESCRIPTION = """
# 🕺 IRG-MotionLLM — Text-to-3D-Motion
Generate 3D human motion from a text description with
**[IRG-MotionLLM](https://huggingface.co/papers/2512.10730)** — an LLM
(Gemma-2-2B) that *interleaves* motion generation, self-assessment and
refinement, then decodes discrete motion tokens through a VQ-VAE into a
HumanML3D skeleton animation.
"""
EXAMPLES = [
["a person walks forward, then turns around and walks back.", True, 1300, 0],
["a man is doing cartwheels.", True, 1300, 0],
["a person jumps up high with both hands raised.", True, 1300, 0],
["someone sits down on a chair and crosses their legs.", True, 1300, 0],
["a person raises their right hand and waves.", True, 1300, 0],
]
with gr.Blocks(theme=gr.themes.Citrus()) as demo:
gr.Markdown(DESCRIPTION)
with gr.Row():
with gr.Column(scale=1):
caption = gr.Textbox(
label="Motion description",
placeholder="e.g. a person walks forward, then turns around and walks back.",
lines=2,
)
run_btn = gr.Button("Generate motion", variant="primary")
with gr.Accordion("Advanced options", open=False):
reasoning = gr.Checkbox(
value=True,
label="Interleaved reasoning (generate → assess → refine)",
info="Uses the full IRG chain-of-thought. Turn off for a faster, direct generation.",
)
max_new_tokens = gr.Slider(
400, 2000, value=1300, step=50,
label="Max new tokens (reasoning budget)",
)
seed = gr.Slider(
0, 100000, value=0, step=1,
label="Seed (0 = greedy / deterministic, >0 = sampling)",
)
with gr.Column(scale=1):
video_out = gr.Video(label="Generated motion", autoplay=True)
info_out = gr.Textbox(label="Summary", lines=2)
with gr.Accordion("Model reasoning trace", open=False):
trace_out = gr.Textbox(label="Interleaved generation / assessment / refinement", lines=10)
gr.Examples(
examples=EXAMPLES,
inputs=[caption, reasoning, max_new_tokens, seed],
outputs=[video_out, info_out, trace_out],
fn=generate,
cache_examples=True,
cache_mode="lazy",
)
run_btn.click(
fn=generate,
inputs=[caption, reasoning, max_new_tokens, seed],
outputs=[video_out, info_out, trace_out],
)
caption.submit(
fn=generate,
inputs=[caption, reasoning, max_new_tokens, seed],
outputs=[video_out, info_out, trace_out],
)
if __name__ == "__main__":
demo.queue().launch()