Spaces:
Sleeping
Sleeping
| """ | |
| app.py — mocap-claude / GameMaster Mocap | |
| Dynamic preview version with normal-standing idle preview. | |
| Workflow: | |
| 1. Upload a single-person video. | |
| 2. Process video once: extract frames -> pose detection -> MotionBERT -> cache poses_3d. | |
| 3. Move motion-strength sliders: only rerun retarget/export preview, not the full AI mocap pipeline. | |
| 4. Export final GLB / optional FBX / TRES / RES using the final slider values. | |
| This version fixes: | |
| - blank custom HTML viewport by using gr.Model3D | |
| - Checkbox .release() crash by using .change() for checkboxes | |
| - startup preview T-pose by generating a separate static idle-preview GLB | |
| - repeated full mocap reruns when only motion strength changes | |
| """ | |
| from __future__ import annotations | |
| import importlib | |
| import logging | |
| import os | |
| import shutil | |
| import sys | |
| import time | |
| import traceback | |
| from typing import Any, Dict, Optional, Tuple | |
| try: | |
| from dotenv import load_dotenv | |
| load_dotenv() | |
| except Exception: | |
| pass | |
| import gradio as gr | |
| APP_DIR = os.path.dirname(os.path.abspath(__file__)) | |
| if APP_DIR not in sys.path: | |
| sys.path.insert(0, APP_DIR) | |
| from pipeline.frame_extractor import extract_frames | |
| from pipeline.pose_detector import detect_poses | |
| from pipeline.motionbert_runner import run_motionbert | |
| from pipeline.godot_exporter import export_godot_resources | |
| from pipeline.fbx_exporter import export_fbx_from_glb | |
| from utils.postprocess import full_postprocess | |
| import pipeline.glb_exporter as glb_exporter | |
| logging.basicConfig( | |
| level=logging.INFO, | |
| format="%(asctime)s %(levelname)-8s %(message)s", | |
| datefmt="%H:%M:%S", | |
| ) | |
| log = logging.getLogger(__name__) | |
| HF_TOKEN = os.environ.get("HF_TOKEN") | |
| OUTPUT_ROOT = os.environ.get("MOCAP_OUTPUT_ROOT", "/tmp/mocap_outputs") | |
| os.makedirs(OUTPUT_ROOT, exist_ok=True) | |
| _IDLE_PREVIEW_CACHE: Optional[str] = None | |
| # --------------------------------------------------------- | |
| # Basic helpers | |
| # --------------------------------------------------------- | |
| def get_template_path() -> str: | |
| return os.environ.get( | |
| "CHARACTER_TEMPLATE_PATH", | |
| os.path.join(APP_DIR, "assets", "character_template.glb"), | |
| ) | |
| def bool_to_env(value: bool) -> str: | |
| return "1" if bool(value) else "0" | |
| def set_idle_env_defaults() -> None: | |
| """ | |
| Defaults for both the initial static preview and the first animation frame. | |
| These are set only when the user has not already set the same value in | |
| HF Space Settings > Variables. | |
| """ | |
| os.environ.setdefault("MOCAP_START_WITH_IDLE_POSE", "1") | |
| os.environ.setdefault("MOCAP_IDLE_ARMS_DOWN", "1") | |
| os.environ.setdefault("MOCAP_IDLE_TRANSITION_FRAMES", "6") | |
| os.environ.setdefault("MOCAP_IDLE_TORSO_UPRIGHT", "0") | |
| os.environ.setdefault("MOCAP_IDLE_LEGS_STRAIGHT", "0") | |
| os.environ.setdefault("MOCAP_IDLE_ARM_SIDE_OFFSET", "0.08") | |
| os.environ.setdefault("MOCAP_IDLE_LEG_SIDE_OFFSET", "0.035") | |
| def get_initial_preview_model() -> Optional[str]: | |
| """ | |
| Return a normal-standing idle GLB for the startup Model3D viewport. | |
| Important: | |
| - It does NOT overwrite assets/character_template.glb. | |
| - It creates /tmp/mocap_outputs/template_idle_preview.glb once. | |
| - If anything fails, it falls back to the raw template instead of crashing. | |
| """ | |
| global _IDLE_PREVIEW_CACHE | |
| template = get_template_path() | |
| if not os.path.exists(template): | |
| return None | |
| if _IDLE_PREVIEW_CACHE and os.path.exists(_IDLE_PREVIEW_CACHE): | |
| return _IDLE_PREVIEW_CACHE | |
| set_idle_env_defaults() | |
| try: | |
| # Reload so glb_exporter sees the idle defaults/env settings. | |
| importlib.reload(glb_exporter) | |
| idle_path = os.path.join(OUTPUT_ROOT, "template_idle_preview.glb") | |
| glb_exporter.export_idle_preview_glb( | |
| template_path=template, | |
| output_path=idle_path, | |
| clear_animations=True, | |
| ) | |
| if os.path.exists(idle_path): | |
| _IDLE_PREVIEW_CACHE = idle_path | |
| return idle_path | |
| except Exception: | |
| log.warning("Could not create idle preview GLB; falling back to raw template.\n%s", traceback.format_exc()) | |
| return template | |
| def normalize_video_input(video_input: Any) -> Optional[str]: | |
| if video_input is None: | |
| return None | |
| if isinstance(video_input, str): | |
| return video_input | |
| if isinstance(video_input, dict): | |
| for key in ("path", "name", "video"): | |
| value = video_input.get(key) | |
| if isinstance(value, str) and value: | |
| return value | |
| return None | |
| def file_size_kb(path: Optional[str]) -> float: | |
| if not path: | |
| return 0.0 | |
| if not os.path.exists(path): | |
| return 0.0 | |
| return os.path.getsize(path) / 1024.0 | |
| def timestamped_dir(prefix: str) -> str: | |
| stamp = time.strftime("%Y%m%d_%H%M%S") | |
| unique = f"{prefix}_{stamp}_{int(time.time() * 1000) % 1000000}" | |
| path = os.path.join(OUTPUT_ROOT, unique) | |
| os.makedirs(path, exist_ok=True) | |
| return path | |
| def set_motion_env( | |
| rotation_strength: float, | |
| root_motion_strength: float, | |
| arm_strength: float, | |
| leg_strength: float, | |
| torso_strength: float, | |
| head_strength: float, | |
| hand_foot_strength: float, | |
| limit_multiplier: float, | |
| animate_hips: bool, | |
| disable_rotation_limits: bool, | |
| ) -> None: | |
| """ | |
| pipeline/glb_exporter.py reads these variables at import time. | |
| Therefore, after setting these, we reload glb_exporter before exporting. | |
| """ | |
| set_idle_env_defaults() | |
| os.environ["MOCAP_ROTATION_STRENGTH"] = str(float(rotation_strength)) | |
| os.environ["MOCAP_ROOT_MOTION_STRENGTH"] = str(float(root_motion_strength)) | |
| os.environ["MOCAP_ARM_STRENGTH"] = str(float(arm_strength)) | |
| os.environ["MOCAP_LEG_STRENGTH"] = str(float(leg_strength)) | |
| os.environ["MOCAP_TORSO_STRENGTH"] = str(float(torso_strength)) | |
| os.environ["MOCAP_HEAD_STRENGTH"] = str(float(head_strength)) | |
| os.environ["MOCAP_HAND_FOOT_STRENGTH"] = str(float(hand_foot_strength)) | |
| os.environ["MOCAP_LIMIT_MULTIPLIER"] = str(float(limit_multiplier)) | |
| os.environ["MOCAP_ANIMATE_HIPS"] = bool_to_env(animate_hips) | |
| os.environ["MOCAP_DISABLE_ROTATION_LIMITS"] = bool_to_env(disable_rotation_limits) | |
| def check_blender_available() -> str: | |
| blender_path = shutil.which("blender") | |
| if not blender_path: | |
| return "Blender not found. FBX export will be skipped." | |
| try: | |
| import subprocess | |
| result = subprocess.run( | |
| [blender_path, "--version"], | |
| capture_output=True, | |
| text=True, | |
| timeout=20, | |
| ) | |
| output = result.stdout or result.stderr or "" | |
| first_line = output.strip().splitlines()[0] if output.strip() else "Blender found." | |
| return f"Blender found at {blender_path}: {first_line}" | |
| except Exception as exc: | |
| return f"Blender exists but failed to run: {exc}" | |
| def build_motion_settings_status( | |
| rotation_strength: float, | |
| root_motion_strength: float, | |
| arm_strength: float, | |
| leg_strength: float, | |
| torso_strength: float, | |
| head_strength: float, | |
| hand_foot_strength: float, | |
| limit_multiplier: float, | |
| animate_hips: bool, | |
| disable_rotation_limits: bool, | |
| ) -> str: | |
| lines = [ | |
| "### Current motion settings", | |
| "", | |
| f"- Global rotation strength: {float(rotation_strength):.2f}", | |
| f"- Root/body motion strength: {float(root_motion_strength):.2f}", | |
| f"- Arm strength: {float(arm_strength):.2f}", | |
| f"- Leg strength: {float(leg_strength):.2f}", | |
| f"- Torso strength: {float(torso_strength):.2f}", | |
| f"- Head strength: {float(head_strength):.2f}", | |
| f"- Hand/foot strength: {float(hand_foot_strength):.2f}", | |
| f"- Rotation limit multiplier: {float(limit_multiplier):.2f}", | |
| f"- Animate hips/pelvis: {bool(animate_hips)}", | |
| f"- Disable rotation limits: {bool(disable_rotation_limits)}", | |
| f"- Start animation from idle pose: {os.environ.get('MOCAP_START_WITH_IDLE_POSE', '1')}", | |
| f"- Idle transition frames: {os.environ.get('MOCAP_IDLE_TRANSITION_FRAMES', '6')}", | |
| ] | |
| return "\n".join(lines) | |
| def build_success_status( | |
| total_frames: int, | |
| fps: float, | |
| width: int, | |
| height: int, | |
| glb_path: str, | |
| fbx_path: Optional[str], | |
| tres_path: Optional[str], | |
| res_path: Optional[str], | |
| rotation_strength: float, | |
| root_motion_strength: float, | |
| arm_strength: float, | |
| leg_strength: float, | |
| torso_strength: float, | |
| head_strength: float, | |
| hand_foot_strength: float, | |
| limit_multiplier: float, | |
| animate_hips: bool, | |
| disable_rotation_limits: bool, | |
| ) -> str: | |
| fbx_status = "Skipped" | |
| if fbx_path and os.path.exists(fbx_path): | |
| fbx_status = f"Created, {file_size_kb(fbx_path):.1f} KB" | |
| tres_status = "Missing" | |
| if tres_path and os.path.exists(tres_path): | |
| tres_status = f"{file_size_kb(tres_path):.1f} KB" | |
| res_status = "Missing" | |
| if res_path and os.path.exists(res_path): | |
| res_status = f"{file_size_kb(res_path):.1f} KB" | |
| lines = [ | |
| "## Final export complete", | |
| "", | |
| "The exported GLB uses the real rigged character from `assets/character_template.glb`.", | |
| "Frame 0 starts from the generated normal-standing idle pose; later frames follow the mocap deltas.", | |
| "", | |
| "### Output details", | |
| "", | |
| f"- Frames processed: {total_frames}", | |
| f"- FPS: {fps:.2f}", | |
| f"- Duration: {total_frames / max(float(fps), 1e-6):.2f} seconds", | |
| f"- Resolution: {width} x {height}", | |
| f"- Animated character GLB: {file_size_kb(glb_path):.1f} KB", | |
| f"- Optional FBX: {fbx_status}", | |
| f"- Godot `.tres`: {tres_status}", | |
| f"- Godot `.res`: {res_status}", | |
| "", | |
| build_motion_settings_status( | |
| rotation_strength=rotation_strength, | |
| root_motion_strength=root_motion_strength, | |
| arm_strength=arm_strength, | |
| leg_strength=leg_strength, | |
| torso_strength=torso_strength, | |
| head_strength=head_strength, | |
| hand_foot_strength=hand_foot_strength, | |
| limit_multiplier=limit_multiplier, | |
| animate_hips=animate_hips, | |
| disable_rotation_limits=disable_rotation_limits, | |
| ), | |
| "", | |
| "### Tuning tip", | |
| "", | |
| "If movement is too restricted, increase global rotation, arm strength, leg strength, root motion, and rotation limit multiplier.", | |
| "", | |
| "If the mesh twists or folds badly, reduce those values or turn off `Animate hips/pelvis`.", | |
| ] | |
| return "\n".join(lines) | |
| def build_cache_status(cache: Dict[str, Any], preview_status: str) -> str: | |
| total_frames = int(cache.get("total_frames", 0)) | |
| fps = float(cache.get("fps", 0.0)) | |
| width = int(cache.get("width", 0)) | |
| height = int(cache.get("height", 0)) | |
| lines = [ | |
| "## Video processed and mocap cached", | |
| "", | |
| "The expensive stages have finished once. You can now move the sliders to update only the retargeted character preview.", | |
| "", | |
| "### Cached mocap", | |
| "", | |
| f"- Frames: {total_frames}", | |
| f"- FPS: {fps:.2f}", | |
| f"- Resolution: {width} x {height}", | |
| "", | |
| preview_status, | |
| ] | |
| return "\n".join(lines) | |
| def error_status(title: str, tb: str) -> str: | |
| return "\n".join( | |
| [ | |
| f"## {title}", | |
| "", | |
| "```text", | |
| tb, | |
| "```", | |
| "", | |
| "Common checks:", | |
| "", | |
| "1. Make sure `assets/character_template.glb` exists.", | |
| "2. Make sure `pipeline/glb_exporter.py` is the idle-preview/full-motion retargeting version.", | |
| "3. Keep the uploaded video short and single-person.", | |
| "4. If only FBX fails, Blender is missing or failed. The GLB output can still work.", | |
| ] | |
| ) | |
| # --------------------------------------------------------- | |
| # Core retarget/export helpers | |
| # --------------------------------------------------------- | |
| def retarget_to_preview_glb( | |
| poses_3d: Any, | |
| fps: float, | |
| rotation_strength: float, | |
| root_motion_strength: float, | |
| arm_strength: float, | |
| leg_strength: float, | |
| torso_strength: float, | |
| head_strength: float, | |
| hand_foot_strength: float, | |
| limit_multiplier: float, | |
| animate_hips: bool, | |
| disable_rotation_limits: bool, | |
| ) -> str: | |
| template_path = get_template_path() | |
| if not os.path.exists(template_path): | |
| raise RuntimeError( | |
| "Missing rigged character template: " | |
| + template_path | |
| + "\n\nThe Space must contain assets/character_template.glb." | |
| ) | |
| set_motion_env( | |
| rotation_strength=rotation_strength, | |
| root_motion_strength=root_motion_strength, | |
| arm_strength=arm_strength, | |
| leg_strength=leg_strength, | |
| torso_strength=torso_strength, | |
| head_strength=head_strength, | |
| hand_foot_strength=hand_foot_strength, | |
| limit_multiplier=limit_multiplier, | |
| animate_hips=animate_hips, | |
| disable_rotation_limits=disable_rotation_limits, | |
| ) | |
| importlib.reload(glb_exporter) | |
| preview_dir = timestamped_dir("preview") | |
| preview_glb = os.path.join(preview_dir, "preview_character.glb") | |
| glb_exporter.export_glb( | |
| poses_3d, | |
| fps, | |
| output_path=preview_glb, | |
| template_path=template_path, | |
| ) | |
| if not os.path.exists(preview_glb): | |
| raise RuntimeError("Preview GLB was not created: " + preview_glb) | |
| return preview_glb | |
| # --------------------------------------------------------- | |
| # Gradio callback: process video once | |
| # --------------------------------------------------------- | |
| def process_video_once( | |
| video_input: Any, | |
| rotation_strength: float, | |
| root_motion_strength: float, | |
| arm_strength: float, | |
| leg_strength: float, | |
| torso_strength: float, | |
| head_strength: float, | |
| hand_foot_strength: float, | |
| limit_multiplier: float, | |
| animate_hips: bool, | |
| disable_rotation_limits: bool, | |
| smooth_sigma: float, | |
| progress: gr.Progress = gr.Progress(track_tqdm=True), | |
| ) -> Tuple[ | |
| Optional[Dict[str, Any]], | |
| Optional[str], | |
| Optional[str], | |
| str, | |
| ]: | |
| video_path = normalize_video_input(video_input) | |
| if not video_path: | |
| return None, get_initial_preview_model(), None, "Please upload a video first." | |
| try: | |
| template_path = get_template_path() | |
| if not os.path.exists(template_path): | |
| raise RuntimeError( | |
| "Missing rigged character template: " | |
| + template_path | |
| + "\n\nThe Space must contain assets/character_template.glb." | |
| ) | |
| progress(0.00, desc="Extracting frames") | |
| frames, fps = extract_frames( | |
| video_path, | |
| progress_cb=lambda f: progress(f * 0.07, desc="Extracting frames"), | |
| ) | |
| if not frames: | |
| raise RuntimeError("No frames were extracted from the uploaded video.") | |
| total_frames = len(frames) | |
| width = frames[0].width | |
| height = frames[0].height | |
| log.info("Frames: %d @ %.2f fps, %dx%d", total_frames, fps, width, height) | |
| progress(0.07, desc="Detecting 2D pose") | |
| coco_kps = detect_poses( | |
| frames, | |
| hf_token=HF_TOKEN, | |
| progress_cb=lambda f: progress(0.07 + f * 0.35, desc="Detecting 2D pose"), | |
| ) | |
| progress(0.42, desc="Running MotionBERT") | |
| poses_3d = run_motionbert( | |
| coco_kps, | |
| img_w=width, | |
| img_h=height, | |
| progress_cb=lambda f: progress(0.42 + f * 0.28, desc="Lifting to 3D"), | |
| ) | |
| progress(0.70, desc="Smoothing and post-processing") | |
| poses_3d, fps = full_postprocess( | |
| poses_3d, | |
| fps, | |
| smooth_sigma=float(smooth_sigma), | |
| do_centre=True, | |
| do_floor=True, | |
| ) | |
| cache = { | |
| "poses_3d": poses_3d, | |
| "fps": fps, | |
| "width": width, | |
| "height": height, | |
| "total_frames": total_frames, | |
| "smooth_sigma": float(smooth_sigma), | |
| } | |
| progress(0.86, desc="Building first preview") | |
| preview_glb = retarget_to_preview_glb( | |
| poses_3d=poses_3d, | |
| fps=fps, | |
| rotation_strength=rotation_strength, | |
| root_motion_strength=root_motion_strength, | |
| arm_strength=arm_strength, | |
| leg_strength=leg_strength, | |
| torso_strength=torso_strength, | |
| head_strength=head_strength, | |
| hand_foot_strength=hand_foot_strength, | |
| limit_multiplier=limit_multiplier, | |
| animate_hips=animate_hips, | |
| disable_rotation_limits=disable_rotation_limits, | |
| ) | |
| progress(1.0, desc="Done") | |
| preview_status = "\n".join( | |
| [ | |
| "### Preview updated", | |
| "", | |
| "The preview animation starts from a generated normal standing idle pose, then follows mocap deltas.", | |
| "", | |
| f"- Preview GLB: {file_size_kb(preview_glb):.1f} KB", | |
| "", | |
| build_motion_settings_status( | |
| rotation_strength=rotation_strength, | |
| root_motion_strength=root_motion_strength, | |
| arm_strength=arm_strength, | |
| leg_strength=leg_strength, | |
| torso_strength=torso_strength, | |
| head_strength=head_strength, | |
| hand_foot_strength=hand_foot_strength, | |
| limit_multiplier=limit_multiplier, | |
| animate_hips=animate_hips, | |
| disable_rotation_limits=disable_rotation_limits, | |
| ), | |
| ] | |
| ) | |
| status = build_cache_status(cache, preview_status) | |
| return cache, preview_glb, preview_glb, status | |
| except Exception: | |
| tb = traceback.format_exc() | |
| log.error("Processing error:\n%s", tb) | |
| return None, get_initial_preview_model(), None, error_status("Processing error", tb) | |
| # --------------------------------------------------------- | |
| # Gradio callback: update preview from cached mocap | |
| # --------------------------------------------------------- | |
| def update_character_preview_from_cache( | |
| cache: Optional[Dict[str, Any]], | |
| rotation_strength: float, | |
| root_motion_strength: float, | |
| arm_strength: float, | |
| leg_strength: float, | |
| torso_strength: float, | |
| head_strength: float, | |
| hand_foot_strength: float, | |
| limit_multiplier: float, | |
| animate_hips: bool, | |
| disable_rotation_limits: bool, | |
| ) -> Tuple[Optional[str], Optional[str], str]: | |
| if not cache: | |
| return ( | |
| get_initial_preview_model(), | |
| None, | |
| "No cached mocap data yet. Click `1. Process Video` first. The preview is showing the static normal-standing idle pose.", | |
| ) | |
| try: | |
| poses_3d = cache["poses_3d"] | |
| fps = float(cache["fps"]) | |
| preview_glb = retarget_to_preview_glb( | |
| poses_3d=poses_3d, | |
| fps=fps, | |
| rotation_strength=rotation_strength, | |
| root_motion_strength=root_motion_strength, | |
| arm_strength=arm_strength, | |
| leg_strength=leg_strength, | |
| torso_strength=torso_strength, | |
| head_strength=head_strength, | |
| hand_foot_strength=hand_foot_strength, | |
| limit_multiplier=limit_multiplier, | |
| animate_hips=animate_hips, | |
| disable_rotation_limits=disable_rotation_limits, | |
| ) | |
| lines = [ | |
| "## Preview updated", | |
| "", | |
| "Only the retarget/export step was rerun. The AI mocap pipeline was not rerun.", | |
| "The animation starts from normal-standing idle, then follows mocap deltas.", | |
| "", | |
| f"- Preview GLB: {file_size_kb(preview_glb):.1f} KB", | |
| "", | |
| build_motion_settings_status( | |
| rotation_strength=rotation_strength, | |
| root_motion_strength=root_motion_strength, | |
| arm_strength=arm_strength, | |
| leg_strength=leg_strength, | |
| torso_strength=torso_strength, | |
| head_strength=head_strength, | |
| hand_foot_strength=hand_foot_strength, | |
| limit_multiplier=limit_multiplier, | |
| animate_hips=animate_hips, | |
| disable_rotation_limits=disable_rotation_limits, | |
| ), | |
| ] | |
| return preview_glb, preview_glb, "\n".join(lines) | |
| except Exception: | |
| tb = traceback.format_exc() | |
| log.error("Preview error:\n%s", tb) | |
| return get_initial_preview_model(), None, error_status("Preview error", tb) | |
| # --------------------------------------------------------- | |
| # Gradio callback: export final files from cached mocap | |
| # --------------------------------------------------------- | |
| def export_final_from_cache( | |
| cache: Optional[Dict[str, Any]], | |
| rotation_strength: float, | |
| root_motion_strength: float, | |
| arm_strength: float, | |
| leg_strength: float, | |
| torso_strength: float, | |
| head_strength: float, | |
| hand_foot_strength: float, | |
| limit_multiplier: float, | |
| animate_hips: bool, | |
| disable_rotation_limits: bool, | |
| ) -> Tuple[ | |
| Optional[str], | |
| Optional[str], | |
| Optional[str], | |
| Optional[str], | |
| str, | |
| ]: | |
| if not cache: | |
| return None, None, None, None, "No cached mocap data. Click `1. Process Video` first." | |
| try: | |
| poses_3d = cache["poses_3d"] | |
| fps = float(cache["fps"]) | |
| width = int(cache.get("width", 0)) | |
| height = int(cache.get("height", 0)) | |
| total_frames = int(cache.get("total_frames", 0)) | |
| template_path = get_template_path() | |
| if not os.path.exists(template_path): | |
| raise RuntimeError( | |
| "Missing rigged character template: " | |
| + template_path | |
| + "\n\nThe Space must contain assets/character_template.glb." | |
| ) | |
| set_motion_env( | |
| rotation_strength=rotation_strength, | |
| root_motion_strength=root_motion_strength, | |
| arm_strength=arm_strength, | |
| leg_strength=leg_strength, | |
| torso_strength=torso_strength, | |
| head_strength=head_strength, | |
| hand_foot_strength=hand_foot_strength, | |
| limit_multiplier=limit_multiplier, | |
| animate_hips=animate_hips, | |
| disable_rotation_limits=disable_rotation_limits, | |
| ) | |
| importlib.reload(glb_exporter) | |
| out_dir = timestamped_dir("final") | |
| glb_path = os.path.join(out_dir, "motion_capture_character.glb") | |
| fbx_path = os.path.join(out_dir, "motion_capture_character.fbx") | |
| glb_exporter.export_glb( | |
| poses_3d, | |
| fps, | |
| output_path=glb_path, | |
| template_path=template_path, | |
| ) | |
| if not os.path.exists(glb_path): | |
| raise RuntimeError("Final GLB was not created: " + glb_path) | |
| try: | |
| maybe_fbx = export_fbx_from_glb(glb_path, fbx_path) | |
| if maybe_fbx and os.path.exists(maybe_fbx): | |
| fbx_path = maybe_fbx | |
| else: | |
| fbx_path = None | |
| except Exception as exc: | |
| log.warning("FBX export skipped or failed: %s", exc) | |
| fbx_path = None | |
| export_godot_resources( | |
| poses_3d, | |
| fps, | |
| output_dir=out_dir, | |
| anim_name="mocap", | |
| ) | |
| tres_path = os.path.join(out_dir, "animation.tres") | |
| res_path = os.path.join(out_dir, "animation.res") | |
| if not os.path.exists(tres_path): | |
| tres_path = None | |
| if not os.path.exists(res_path): | |
| res_path = None | |
| status = build_success_status( | |
| total_frames=total_frames, | |
| fps=fps, | |
| width=width, | |
| height=height, | |
| glb_path=glb_path, | |
| fbx_path=fbx_path, | |
| tres_path=tres_path, | |
| res_path=res_path, | |
| rotation_strength=rotation_strength, | |
| root_motion_strength=root_motion_strength, | |
| arm_strength=arm_strength, | |
| leg_strength=leg_strength, | |
| torso_strength=torso_strength, | |
| head_strength=head_strength, | |
| hand_foot_strength=hand_foot_strength, | |
| limit_multiplier=limit_multiplier, | |
| animate_hips=animate_hips, | |
| disable_rotation_limits=disable_rotation_limits, | |
| ) | |
| return glb_path, fbx_path, tres_path, res_path, status | |
| except Exception: | |
| tb = traceback.format_exc() | |
| log.error("Final export error:\n%s", tb) | |
| return None, None, None, None, error_status("Final export error", tb) | |
| # --------------------------------------------------------- | |
| # Static UI text | |
| # --------------------------------------------------------- | |
| BONE_TREE = "\n".join( | |
| [ | |
| "The character template keeps its real mesh, materials, skin weights, fingers, and extra bones.", | |
| "", | |
| "The 17 animated mocap target bones are:", | |
| "", | |
| "```text", | |
| "Hips", | |
| "Spine", | |
| "Chest", | |
| "Neck", | |
| "Head", | |
| "LeftUpperArm", | |
| "LeftLowerArm", | |
| "LeftHand", | |
| "RightUpperArm", | |
| "RightLowerArm", | |
| "RightHand", | |
| "LeftUpperLeg", | |
| "LeftLowerLeg", | |
| "LeftFoot", | |
| "RightUpperLeg", | |
| "RightLowerLeg", | |
| "RightFoot", | |
| "```", | |
| ] | |
| ) | |
| GODOT_USAGE = "\n".join( | |
| [ | |
| "1. Import `motion_capture_character.glb` into Godot.", | |
| "2. Open the imported scene and play the embedded `mocap` animation.", | |
| "3. The separate `animation.tres` and `animation.res` files are also exported for Godot animation-library workflows.", | |
| "", | |
| "The animated GLB is usually the safest Godot output because it contains the mesh, armature, skin weights, and embedded animation together.", | |
| "", | |
| "GDScript example:", | |
| "", | |
| "```gdscript", | |
| "func _ready() -> void:", | |
| " var player := $AnimationPlayer", | |
| " player.play(\"mocap\")", | |
| "```", | |
| ] | |
| ) | |
| BLENDER_USAGE = "\n".join( | |
| [ | |
| "1. File > Import > glTF 2.0 > select `motion_capture_character.glb`.", | |
| "2. Select the imported armature/character.", | |
| "3. Open the Timeline or Dope Sheet and play the `mocap` action.", | |
| "4. You should see the uploaded rigged character moving.", | |
| "", | |
| "If the FBX output is missing, install Blender in the Space. The GLB output is still the primary output.", | |
| ] | |
| ) | |
| DEFAULT_TUNING = "\n".join( | |
| [ | |
| "Recommended starting point:", | |
| "", | |
| "- Global rotation strength: 2.35", | |
| "- Root/body motion strength: 2.25", | |
| "- Arm strength: 2.85", | |
| "- Leg strength: 2.55", | |
| "- Torso strength: 1.90", | |
| "- Head strength: 1.55", | |
| "- Hand/foot strength: 2.25", | |
| "- Rotation limit multiplier: 1.45", | |
| "- Animate hips/pelvis: enabled", | |
| "- Disable rotation limits: disabled", | |
| "- Smoothing sigma: 1.5", | |
| "- Start with idle pose: enabled", | |
| "- Idle transition frames: 6", | |
| "", | |
| "For stronger movement, increase global, arm, leg, and root strengths.", | |
| "", | |
| "For extreme motion testing:", | |
| "", | |
| "- Global rotation strength: 3.0 to 4.0", | |
| "- Arm strength: 3.5 to 5.0", | |
| "- Leg strength: 3.0 to 4.5", | |
| "- Root/body motion strength: 3.0", | |
| "- Rotation limit multiplier: 2.5", | |
| "", | |
| "If the animation waits too long in idle before moving, set MOCAP_IDLE_TRANSITION_FRAMES=1 in Space Variables.", | |
| ] | |
| ) | |
| # --------------------------------------------------------- | |
| # Gradio UI | |
| # --------------------------------------------------------- | |
| with gr.Blocks(title="GameMaster Mocap", theme=gr.themes.Soft()) as demo: | |
| mocap_cache = gr.State(None) | |
| gr.Markdown( | |
| "# GameMaster Mocap\n" | |
| "Upload a single-person video, process mocap once, then tune retarget strength with a live animated GLB preview." | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=5): | |
| video_in = gr.Video( | |
| label="Input video, ideally 5 to 15 seconds", | |
| sources=["upload"], | |
| height=320, | |
| ) | |
| gr.Markdown( | |
| "Free CPU Space note: keep clips short. Processing the video is expensive; slider preview updates reuse cached mocap data." | |
| ) | |
| with gr.Accordion("Motion strength controls", open=True): | |
| gr.Markdown( | |
| "Move a slider, then release it. The Space will rerun only retarget/export, not the full AI mocap pipeline." | |
| ) | |
| rotation_strength = gr.Slider( | |
| label="Global rotation strength", | |
| minimum=0.1, | |
| maximum=8.0, | |
| value=2.35, | |
| step=0.05, | |
| ) | |
| root_motion_strength = gr.Slider( | |
| label="Root/body motion strength", | |
| minimum=0.0, | |
| maximum=10.0, | |
| value=2.25, | |
| step=0.05, | |
| ) | |
| with gr.Row(): | |
| arm_strength = gr.Slider( | |
| label="Arm strength", | |
| minimum=0.1, | |
| maximum=10.0, | |
| value=2.85, | |
| step=0.05, | |
| ) | |
| leg_strength = gr.Slider( | |
| label="Leg strength", | |
| minimum=0.1, | |
| maximum=10.0, | |
| value=2.55, | |
| step=0.05, | |
| ) | |
| with gr.Row(): | |
| torso_strength = gr.Slider( | |
| label="Torso strength", | |
| minimum=0.1, | |
| maximum=8.0, | |
| value=1.90, | |
| step=0.05, | |
| ) | |
| head_strength = gr.Slider( | |
| label="Head strength", | |
| minimum=0.1, | |
| maximum=6.0, | |
| value=1.55, | |
| step=0.05, | |
| ) | |
| hand_foot_strength = gr.Slider( | |
| label="Hand/foot strength", | |
| minimum=0.1, | |
| maximum=8.0, | |
| value=2.25, | |
| step=0.05, | |
| ) | |
| limit_multiplier = gr.Slider( | |
| label="Rotation limit multiplier", | |
| minimum=0.25, | |
| maximum=20.0, | |
| value=1.45, | |
| step=0.05, | |
| ) | |
| with gr.Row(): | |
| animate_hips = gr.Checkbox( | |
| label="Animate hips/pelvis", | |
| value=True, | |
| ) | |
| disable_rotation_limits = gr.Checkbox( | |
| label="Disable rotation limits — more motion, more risk", | |
| value=False, | |
| ) | |
| smooth_sigma = gr.Slider( | |
| label="Smoothing sigma", | |
| minimum=0.0, | |
| maximum=5.0, | |
| value=1.5, | |
| step=0.1, | |
| ) | |
| with gr.Accordion("Recommended values", open=False): | |
| gr.Markdown(DEFAULT_TUNING) | |
| with gr.Row(): | |
| process_btn = gr.Button("1. Process Video", variant="primary", size="lg") | |
| update_preview_btn = gr.Button("Update Preview", variant="secondary") | |
| export_btn = gr.Button("2. Export Final Files", variant="secondary") | |
| with gr.Accordion("Animated bone target list", open=False): | |
| gr.Markdown(BONE_TREE) | |
| with gr.Accordion("Godot 4 setup", open=False): | |
| gr.Markdown(GODOT_USAGE) | |
| with gr.Accordion("Blender setup", open=False): | |
| gr.Markdown(BLENDER_USAGE) | |
| with gr.Column(scale=6): | |
| gr.Markdown("## Live Character Preview") | |
| preview_3d = gr.Model3D( | |
| value=get_initial_preview_model(), | |
| label="Live animated GLB preview", | |
| height=520, | |
| ) | |
| preview_glb_file = gr.File( | |
| label="Current preview GLB", | |
| interactive=False, | |
| ) | |
| status_md = gr.Markdown( | |
| "\n".join( | |
| [ | |
| "The viewport starts with a generated normal-standing idle pose, not the raw T-pose template.", | |
| "", | |
| "Upload a video and click `1. Process Video`.", | |
| "", | |
| "First run downloads DETR, ViTPose, and MotionBERT assets if they are not cached.", | |
| "", | |
| check_blender_available(), | |
| ] | |
| ) | |
| ) | |
| gr.Markdown("### Final downloads") | |
| glb_out = gr.File( | |
| label="motion_capture_character.glb — rigged character plus mocap animation", | |
| interactive=False, | |
| ) | |
| fbx_out = gr.File( | |
| label="motion_capture_character.fbx — optional, requires Blender", | |
| interactive=False, | |
| ) | |
| with gr.Row(): | |
| tres_out = gr.File( | |
| label="animation.tres — Godot text AnimationLibrary", | |
| interactive=False, | |
| ) | |
| res_out = gr.File( | |
| label="animation.res — Godot binary AnimationLibrary", | |
| interactive=False, | |
| ) | |
| process_inputs = [ | |
| video_in, | |
| rotation_strength, | |
| root_motion_strength, | |
| arm_strength, | |
| leg_strength, | |
| torso_strength, | |
| head_strength, | |
| hand_foot_strength, | |
| limit_multiplier, | |
| animate_hips, | |
| disable_rotation_limits, | |
| smooth_sigma, | |
| ] | |
| preview_inputs = [ | |
| mocap_cache, | |
| rotation_strength, | |
| root_motion_strength, | |
| arm_strength, | |
| leg_strength, | |
| torso_strength, | |
| head_strength, | |
| hand_foot_strength, | |
| limit_multiplier, | |
| animate_hips, | |
| disable_rotation_limits, | |
| ] | |
| preview_outputs = [ | |
| preview_3d, | |
| preview_glb_file, | |
| status_md, | |
| ] | |
| final_export_inputs = [ | |
| mocap_cache, | |
| rotation_strength, | |
| root_motion_strength, | |
| arm_strength, | |
| leg_strength, | |
| torso_strength, | |
| head_strength, | |
| hand_foot_strength, | |
| limit_multiplier, | |
| animate_hips, | |
| disable_rotation_limits, | |
| ] | |
| process_btn.click( | |
| fn=process_video_once, | |
| inputs=process_inputs, | |
| outputs=[ | |
| mocap_cache, | |
| preview_3d, | |
| preview_glb_file, | |
| status_md, | |
| ], | |
| show_progress="full", | |
| ) | |
| update_preview_btn.click( | |
| fn=update_character_preview_from_cache, | |
| inputs=preview_inputs, | |
| outputs=preview_outputs, | |
| show_progress="minimal", | |
| ) | |
| export_btn.click( | |
| fn=export_final_from_cache, | |
| inputs=final_export_inputs, | |
| outputs=[ | |
| glb_out, | |
| fbx_out, | |
| tres_out, | |
| res_out, | |
| status_md, | |
| ], | |
| show_progress="full", | |
| ) | |
| # --------------------------------------------------------- | |
| # Auto-update preview when controls change | |
| # --------------------------------------------------------- | |
| # Sliders support .release(), so preview updates after the user releases the slider. | |
| # Checkboxes do NOT support .release(); they must use .change(). | |
| # This fixes: | |
| # AttributeError: 'Checkbox' object has no attribute 'release' | |
| # --------------------------------------------------------- | |
| slider_controls = [ | |
| rotation_strength, | |
| root_motion_strength, | |
| arm_strength, | |
| leg_strength, | |
| torso_strength, | |
| head_strength, | |
| hand_foot_strength, | |
| limit_multiplier, | |
| ] | |
| for slider in slider_controls: | |
| slider.release( | |
| fn=update_character_preview_from_cache, | |
| inputs=preview_inputs, | |
| outputs=preview_outputs, | |
| show_progress="minimal", | |
| ) | |
| checkbox_controls = [ | |
| animate_hips, | |
| disable_rotation_limits, | |
| ] | |
| for checkbox in checkbox_controls: | |
| checkbox.change( | |
| fn=update_character_preview_from_cache, | |
| inputs=preview_inputs, | |
| outputs=preview_outputs, | |
| show_progress="minimal", | |
| ) | |
| gr.Markdown( | |
| "---\n" | |
| "Uses DETR, ViTPose, and MotionBERT for pose extraction/lifting, then retargets onto the provided rigged GLB template." | |
| ) | |
| if __name__ == "__main__": | |
| set_idle_env_defaults() | |
| demo.launch( | |
| server_name="0.0.0.0", | |
| server_port=7860, | |
| share=False, | |
| allowed_paths=[ | |
| OUTPUT_ROOT, | |
| os.path.join(APP_DIR, "assets"), | |
| ], | |
| ) | |