Spaces:
Paused
Paused
| import base64 | |
| import html | |
| import math | |
| import os | |
| from pathlib import Path | |
| import tempfile | |
| from typing import Any, List, Optional, Tuple, Union | |
| import gradio as gr | |
| import numpy as np | |
| import spaces | |
| import torch | |
| from media_loader import load_media_views | |
| from zipsplat import ZipSplat, viz | |
| from zipsplat.camera import Camera | |
| from zipsplat.gaussians import Gaussians | |
| # ---------------------------------------------------------------------------- | |
| # Viewer HTML Template Loader | |
| # ---------------------------------------------------------------------------- | |
| VIEWER_HTML_PATH = Path(__file__).parent / "static" / "viewer" / "viewer.html" | |
| def get_viewer_template() -> str: | |
| if VIEWER_HTML_PATH.exists(): | |
| return VIEWER_HTML_PATH.read_text(encoding="utf-8") | |
| return "" | |
| def build_viewer_iframe(ply_path: str, cx: float = 0, cy: float = 0, cz: float = 0, r: float = 2) -> str: | |
| template = get_viewer_template() | |
| if not template: | |
| return ( | |
| '<div style="height: 500px; display: flex; align-items: center; justify-content: center; ' | |
| 'background: #080b12; color: #f87171; border-radius: 12px; font-family: system-ui, sans-serif;">' | |
| "Error: Viewer template file static/viewer/viewer.html not found on server.</div>" | |
| ) | |
| try: | |
| with open(ply_path, "rb") as f: | |
| b64_ply = base64.b64encode(f.read()).decode("utf-8") | |
| except Exception as e: | |
| return ( | |
| f'<div style="height: 500px; display: flex; align-items: center; justify-content: center; ' | |
| f'background: #080b12; color: #f87171; border-radius: 12px; font-family: system-ui, sans-serif;">' | |
| f"Error reading PLY file for viewer: {e}</div>" | |
| ) | |
| config_script = f""" | |
| <script> | |
| window.PLY_CONFIG = {{ | |
| plyB64: "{b64_ply}", | |
| cx: {cx}, | |
| cy: {cy}, | |
| cz: {cz}, | |
| r: {r} | |
| }}; | |
| </script> | |
| """ | |
| if "</head>" in template: | |
| full_html = template.replace("</head>", f"{config_script}\n</head>", 1) | |
| else: | |
| full_html = config_script + template | |
| escaped_html = html.escape(full_html, quote=True) | |
| return f""" | |
| <iframe srcdoc="{escaped_html}" | |
| style="width: 100%; height: 500px; border: none; border-radius: 12px; background: #080b12;"> | |
| </iframe> | |
| """ | |
| def render_stroke_turntable( | |
| gaussians: Gaussians, | |
| path: str, | |
| gaussians_per_stroke: int = 32, | |
| num_frames: int = 180, | |
| fps: int = 30, | |
| render_size: int = 512, | |
| bg: Tuple[float, float, float] = (1.0, 1.0, 1.0), | |
| ) -> str: | |
| """Render a video showing the 3D scene painted stroke-by-stroke during orbit. | |
| As the camera orbits, strokes (groups of `gaussians_per_stroke` Gaussians) are progressively | |
| added frame by frame, revealing the 3D structure from initial strokes to full detail. | |
| """ | |
| device = gaussians.device | |
| fov = torch.tensor(math.radians(55.0)) | |
| camera = Camera.from_fov(fov, w=render_size, h=render_size).to(device) | |
| center, radius = viz.scene_center_radius(gaussians) | |
| poses = viz.orbit_poses(center, radius, num_frames, sweep_deg=360.0) | |
| cameras = Camera(camera.data_.unsqueeze(0).expand(num_frames, -1).clone()).to(device) | |
| poses = poses.to(device) | |
| total_gaussians = gaussians.num_gaussians | |
| total_strokes = math.ceil(total_gaussians / gaussians_per_stroke) | |
| reveal_frames = int(num_frames * 0.8) | |
| bg_t = torch.tensor(bg, device=device, dtype=torch.float32) | |
| frames = [] | |
| for f in range(num_frames): | |
| if f >= reveal_frames: | |
| stroke_count = total_strokes | |
| else: | |
| stroke_count = max(1, math.ceil(((f + 1) / reveal_frames) * total_strokes)) | |
| num_visible = min(stroke_count * gaussians_per_stroke, total_gaussians) | |
| sub_g = Gaussians(gaussians.data_[:num_visible]) | |
| rgb, _ = sub_g.render( | |
| cameras[f : f + 1], poses[f : f + 1], mode="RGB", backgrounds=bg_t.expand(1, 3) | |
| ) | |
| rgb = rgb.float().clamp(0, 1).cpu() | |
| frame_np = (rgb.permute(0, 2, 3, 1).numpy() * 255).astype(np.uint8)[0] | |
| frames.append(frame_np) | |
| frames_np = np.stack(frames, axis=0) | |
| viz.save_video(frames_np, path, fps=fps) | |
| return path | |
| def normalize_uploaded_paths(files: Any) -> List[Path]: | |
| """Normalize Gradio return types (None, str, Path, list, dict, file objects) to a list of Path objects.""" | |
| if files is None: | |
| return [] | |
| if isinstance(files, (str, Path)): | |
| return [Path(files)] | |
| if isinstance(files, dict): | |
| p = files.get("name") or files.get("path") | |
| orig = files.get("orig_name") | |
| if p: | |
| path_obj = Path(p) | |
| if orig and not path_obj.suffix: | |
| orig_suffix = Path(orig).suffix | |
| if orig_suffix: | |
| new_path = path_obj.with_name(path_obj.name + orig_suffix) | |
| if not new_path.exists() and path_obj.exists(): | |
| try: | |
| os.symlink(path_obj, new_path) | |
| return [new_path] | |
| except Exception: | |
| pass | |
| return [path_obj] | |
| return [] | |
| paths = [] | |
| if isinstance(files, (list, tuple)): | |
| for item in files: | |
| paths.extend(normalize_uploaded_paths(item)) | |
| elif hasattr(files, "name"): | |
| p = files.name | |
| orig = getattr(files, "orig_name", None) | |
| path_obj = Path(p) | |
| if orig and not path_obj.suffix: | |
| orig_suffix = Path(orig).suffix | |
| if orig_suffix: | |
| new_path = path_obj.with_name(path_obj.name + orig_suffix) | |
| if not new_path.exists() and path_obj.exists(): | |
| try: | |
| os.symlink(path_obj, new_path) | |
| return [new_path] | |
| except Exception: | |
| pass | |
| paths.append(path_obj) | |
| return paths | |
| # ---------------------------------------------------------------------------- | |
| # ZipSplat Model Initialization (loaded CPU-side, ZeroGPU moves to GPU) | |
| # ---------------------------------------------------------------------------- | |
| model = ZipSplat(weights="zipsplat").eval() | |
| def generate(files: Any): | |
| paths = normalize_uploaded_paths(files) | |
| if not paths: | |
| default_placeholder = ( | |
| '<div style="height: 500px; display: flex; align-items: center; justify-content: center; ' | |
| 'background: #080b12; color: #94a3b8; border-radius: 12px; font-family: system-ui, sans-serif;">' | |
| "Please upload images, videos, or Live Photo pairs.</div>" | |
| ) | |
| return None, None, None, default_placeholder, "No files uploaded." | |
| try: | |
| result = load_media_views(paths, max_views=500, frames_per_video=8) | |
| except Exception as e: | |
| err_msg = f"Error loading media: {e}" | |
| err_placeholder = ( | |
| f'<div style="height: 500px; display: flex; align-items: center; justify-content: center; ' | |
| f'background: #080b12; color: #f87171; border-radius: 12px; font-family: system-ui, sans-serif;">' | |
| f"{err_msg}</div>" | |
| ) | |
| return None, None, None, err_placeholder, err_msg | |
| if not result.images: | |
| status = "Error: No valid views could be loaded." | |
| if result.warnings: | |
| status += "\n\nWarnings:\n" + "\n".join(result.warnings) | |
| err_placeholder = ( | |
| f'<div style="height: 500px; display: flex; align-items: center; justify-content: center; ' | |
| f'background: #080b12; color: #f87171; border-radius: 12px; font-family: system-ui, sans-serif;">' | |
| f"{status}</div>" | |
| ) | |
| return None, None, None, err_placeholder, status | |
| import traceback | |
| try: | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| model.to(device) | |
| gaussians = model(result.images)[0] | |
| video_path = tempfile.mktemp(suffix=".mp4") | |
| stroke_video_path = tempfile.mktemp(suffix="_stroke.mp4") | |
| ply_path = tempfile.mktemp(suffix=".ply") | |
| viz.turntable(gaussians, video_path, sweep_deg=None) | |
| render_stroke_turntable(gaussians, stroke_video_path) | |
| gaussians.save_ply(ply_path) | |
| center, radius = viz.scene_center_radius(gaussians) | |
| cx, cy, cz = center.tolist() | |
| viewer_html = build_viewer_iframe(ply_path, cx=cx, cy=cy, cz=cz, r=radius) | |
| total_g = gaussians.num_gaussians | |
| total_s = math.ceil(total_g / 32) | |
| status = ( | |
| f"Loaded {result.image_count} still image(s) and selected " | |
| f"{result.selected_video_frames} of {result.decoded_video_frames} decoded video frame(s).\n" | |
| f"Total ZipSplat views: {len(result.images)}.\n" | |
| f"Generated 3D scene with {total_g:,} Gaussians across {total_s:,} strokes (32 Gaussians/stroke)." | |
| ) | |
| if result.warnings: | |
| status += "\n\nWarnings:\n" + "\n".join(result.warnings) | |
| return video_path, stroke_video_path, ply_path, viewer_html, status | |
| except Exception as e: | |
| err_msg = f"Backend Crash during generation:\n{traceback.format_exc()}" | |
| err_html = ( | |
| f'<div style="height: 500px; padding: 20px; overflow-y: auto; ' | |
| f'background: #080b12; color: #f87171; border-radius: 12px; font-family: monospace;">' | |
| f"{err_msg.replace(chr(10), '<br>')}</div>" | |
| ) | |
| return None, None, None, err_html, err_msg | |
| # ---------------------------------------------------------------------------- | |
| # SEO Head Metadata & Social OpenGraph Tags | |
| # ---------------------------------------------------------------------------- | |
| SEO_HEAD_HTML = """ | |
| <meta name="description" content="ZipSplatPlus: Instant single & multi-image 3D Gaussian Splatting with real-time stroke-by-stroke interactive 3D WebGL viewer on ZeroGPU. Convert photos, videos, and Apple Live Photos into 3D scenes!" /> | |
| <meta name="keywords" content="ZipSplat, ZipSplatPlus, 3D Gaussian Splatting, Stroke Playback, 3D Reconstruction, Single Image to 3D, Novel View Synthesis, Spark.js, ZeroGPU, HEIC 3D, Live Photo 3D, WebGL 3D Viewer" /> | |
| <meta name="author" content="ZipSplat Team" /> | |
| <meta name="robots" content="index, follow, max-image-preview:large" /> | |
| <link rel="canonical" href="https://huggingface.co/spaces/yosun/ZipSplatPlus" /> | |
| <!-- Open Graph / Facebook / LinkedIn / Slack / Discord --> | |
| <meta property="og:type" content="website" /> | |
| <meta property="og:site_name" content="ZipSplatPlus" /> | |
| <meta property="og:title" content="ZipSplatPlus: Real-time 3D Gaussian Splatting & Stroke Playback" /> | |
| <meta property="og:description" content="Convert images, videos & Apple Live Photos to interactive 3D Gaussian Splatting scenes with stroke-by-stroke playback." /> | |
| <meta property="og:image" content="https://huggingface.co/spaces/yosun/ZipSplatPlus/resolve/main/thumbnail.png" /> | |
| <meta property="og:url" content="https://huggingface.co/spaces/yosun/ZipSplatPlus" /> | |
| <!-- Twitter / X Cards --> | |
| <meta name="twitter:card" content="summary_large_image" /> | |
| <meta name="twitter:title" content="ZipSplatPlus: Real-time 3D Gaussian Splatting & Stroke Playback" /> | |
| <meta name="twitter:description" content="Instant 3D reconstruction with stroke-by-stroke playback powered by ZipSplat + Spark.js interactive viewer." /> | |
| <meta name="twitter:image" content="https://huggingface.co/spaces/yosun/ZipSplatPlus/resolve/main/thumbnail.png" /> | |
| <!-- Schema.org JSON-LD Microdata for Google Search Rich Results --> | |
| <script type="application/ld+json"> | |
| { | |
| "@context": "https://schema.org", | |
| "@type": "WebApplication", | |
| "name": "ZipSplatPlus", | |
| "alternateName": "ZipSplat 3D Gaussian Splatting Demo", | |
| "url": "https://huggingface.co/spaces/yosun/ZipSplatPlus", | |
| "description": "State-of-the-art 3D Gaussian Splatting web application with interactive stroke-by-stroke 3D viewer, stroke video playback, 360° turntable renderer, and PLY exporter.", | |
| "applicationCategory": "MultimediaApplication", | |
| "operatingSystem": "All", | |
| "image": "https://huggingface.co/spaces/yosun/ZipSplatPlus/resolve/main/thumbnail.png", | |
| "offers": { | |
| "@type": "Offer", | |
| "price": "0", | |
| "priceCurrency": "USD" | |
| }, | |
| "author": { | |
| "@type": "Organization", | |
| "name": "ZipSplat Team" | |
| } | |
| } | |
| </script> | |
| """ | |
| # ---------------------------------------------------------------------------- | |
| # Gradio UI Layout | |
| # ---------------------------------------------------------------------------- | |
| with gr.Blocks( | |
| title="ZipSplatPlus: Real-time 3D Gaussian Splatting & Stroke Playback Viewer", | |
| head=SEO_HEAD_HTML, | |
| ) as demo: | |
| gr.Markdown("# 🔥 ZipSplatPlus: Real-time 3D Gaussian Splatting & Stroke Playback") | |
| gr.Markdown( | |
| "ZeroGPU Powered Demo for [veichta/zipsplat](https://huggingface.co/veichta/zipsplat). " | |
| "Upload images (JPEG, PNG, WebP, HEIC/HEIF), videos (MOV, MP4, M4V), or Apple Live Photo pairs " | |
| "(HEIC + MOV) to generate a 3D Gaussian Splatting scene and **play back each stroke step-by-step** in real time!" | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| media_in = gr.File( | |
| label="Upload Images, Videos, or Live Photo Pairs", | |
| file_count="multiple", | |
| type="filepath", | |
| file_types=[ | |
| ".jpg", | |
| ".jpeg", | |
| ".png", | |
| ".webp", | |
| ".heic", | |
| ".heif", | |
| ".mov", | |
| ".mp4", | |
| ".m4v", | |
| ], | |
| ) | |
| btn = gr.Button("🚀 Generate 3D Scene", variant="primary") | |
| with gr.Column(scale=2): | |
| viewer_out = gr.HTML( | |
| label="Interactive 3D Gaussian & Stroke Viewer (Spark.js)", | |
| value='<div style="height: 500px; display: flex; align-items: center; justify-content: center; background: #080b12; color: #94a3b8; border-radius: 12px; font-family: system-ui, sans-serif;">Upload media and click "Generate 3D Scene" to view interactive 3D Gaussian Splat with stroke-by-stroke playback</div>', | |
| ) | |
| with gr.Tabs(): | |
| with gr.TabItem("🖌️ Stroke-by-Stroke Playback Video"): | |
| stroke_video_out = gr.Video(label="Stroke-by-Stroke Playback Video") | |
| with gr.TabItem("🎬 360° Turntable Video"): | |
| video_out = gr.Video(label="360° Turntable Video Render") | |
| with gr.Row(): | |
| file_out = gr.File(label="Download 3D Model (.ply)") | |
| status_out = gr.Textbox(label="Status & Warnings", interactive=False) | |
| btn.click( | |
| fn=generate, | |
| inputs=[media_in], | |
| outputs=[video_out, stroke_video_out, file_out, viewer_out, status_out], | |
| js="""function(files) { | |
| if (!files || (Array.isArray(files) && files.length === 0)) { | |
| alert("Please upload at least one image or video before generating."); | |
| throw new Error("No files uploaded"); | |
| } | |
| return [files]; | |
| }""", | |
| ) | |
| if __name__ == "__main__": | |
| demo.launch(allowed_paths=[tempfile.gettempdir()]) |