app.py

import base64
import html
import math
import os
from pathlib import Path
import tempfile
from typing import Any, List, Optional, Tuple, Union
import gradio as gr
import numpy as np
import spaces
import torch
from media_loader import load_media_views
from zipsplat import ZipSplat, viz
from zipsplat.camera import Camera
from zipsplat.gaussians import Gaussians
# ----------------------------------------------------------------------------
# Viewer HTML Template Loader
# ----------------------------------------------------------------------------
VIEWER_HTML_PATH = Path(__file__).parent / "static" / "viewer" / "viewer.html"
def get_viewer_template() -> str:
    if VIEWER_HTML_PATH.exists():
        return VIEWER_HTML_PATH.read_text(encoding="utf-8")
    return ""
def build_viewer_iframe(ply_path: str, cx: float = 0, cy: float = 0, cz: float = 0, r: float = 2) -> str:
    template = get_viewer_template()
    if not template:
        return (
            '<div style="height: 500px; display: flex; align-items: center; justify-content: center; '
            'background: #080b12; color: #f87171; border-radius: 12px; font-family: system-ui, sans-serif;">'
            "Error: Viewer template file static/viewer/viewer.html not found on server.</div>"
        )
    try:
        with open(ply_path, "rb") as f:
            b64_ply = base64.b64encode(f.read()).decode("utf-8")
    except Exception as e:
        return (
            f'<div style="height: 500px; display: flex; align-items: center; justify-content: center; '
            f'background: #080b12; color: #f87171; border-radius: 12px; font-family: system-ui, sans-serif;">'
            f"Error reading PLY file for viewer: {e}</div>"
        )
    config_script = f"""
    <script>
      window.PLY_CONFIG = {{
        plyB64: "{b64_ply}",
        cx: {cx},
        cy: {cy},
        cz: {cz},
        r: {r}
      }};
    </script>
    """
    if "</head>" in template:
        full_html = template.replace("</head>", f"{config_script}\n</head>", 1)
    else:
        full_html = config_script + template
    escaped_html = html.escape(full_html, quote=True)
    return f"""
    <iframe srcdoc="{escaped_html}"
            style="width: 100%; height: 500px; border: none; border-radius: 12px; background: #080b12;">
    </iframe>
    """
def render_stroke_turntable(
    gaussians: Gaussians,
    path: str,
    gaussians_per_stroke: int = 32,
    num_frames: int = 180,
    fps: int = 30,
    render_size: int = 512,
    bg: Tuple[float, float, float] = (1.0, 1.0, 1.0),
) -> str:
    """Render a video showing the 3D scene painted stroke-by-stroke during orbit.
    As the camera orbits, strokes (groups of `gaussians_per_stroke` Gaussians) are progressively
    added frame by frame, revealing the 3D structure from initial strokes to full detail.
    """
    device = gaussians.device
    fov = torch.tensor(math.radians(55.0))
    camera = Camera.from_fov(fov, w=render_size, h=render_size).to(device)
    center, radius = viz.scene_center_radius(gaussians)
    poses = viz.orbit_poses(center, radius, num_frames, sweep_deg=360.0)
    cameras = Camera(camera.data_.unsqueeze(0).expand(num_frames, -1).clone()).to(device)
    poses = poses.to(device)
    total_gaussians = gaussians.num_gaussians
    total_strokes = math.ceil(total_gaussians / gaussians_per_stroke)
    reveal_frames = int(num_frames * 0.8)
    bg_t = torch.tensor(bg, device=device, dtype=torch.float32)
    frames = []
    for f in range(num_frames):
        if f >= reveal_frames:
            stroke_count = total_strokes
        else:
            stroke_count = max(1, math.ceil(((f + 1) / reveal_frames) * total_strokes))
        num_visible = min(stroke_count * gaussians_per_stroke, total_gaussians)
        sub_g = Gaussians(gaussians.data_[:num_visible])
        rgb, _ = sub_g.render(
            cameras[f : f + 1], poses[f : f + 1], mode="RGB", backgrounds=bg_t.expand(1, 3)
        )
        rgb = rgb.float().clamp(0, 1).cpu()
        frame_np = (rgb.permute(0, 2, 3, 1).numpy() * 255).astype(np.uint8)[0]
        frames.append(frame_np)
    frames_np = np.stack(frames, axis=0)
    viz.save_video(frames_np, path, fps=fps)
    return path
def normalize_uploaded_paths(files: Any) -> List[Path]:
    """Normalize Gradio return types (None, str, Path, list, dict, file objects) to a list of Path objects."""
    if files is None:
        return []
    if isinstance(files, (str, Path)):
        return [Path(files)]
    if isinstance(files, dict):
        p = files.get("name") or files.get("path")
        orig = files.get("orig_name")
        if p:
            path_obj = Path(p)
            if orig and not path_obj.suffix:
                orig_suffix = Path(orig).suffix
                if orig_suffix:
                    new_path = path_obj.with_name(path_obj.name + orig_suffix)
                    if not new_path.exists() and path_obj.exists():
                        try:
                            os.symlink(path_obj, new_path)
                            return [new_path]
                        except Exception:
                            pass
            return [path_obj]
        return []
    paths = []
    if isinstance(files, (list, tuple)):
        for item in files:
            paths.extend(normalize_uploaded_paths(item))
    elif hasattr(files, "name"):
        p = files.name
        orig = getattr(files, "orig_name", None)
        path_obj = Path(p)
        if orig and not path_obj.suffix:
            orig_suffix = Path(orig).suffix
            if orig_suffix:
                new_path = path_obj.with_name(path_obj.name + orig_suffix)
                if not new_path.exists() and path_obj.exists():
                    try:
                        os.symlink(path_obj, new_path)
                        return [new_path]
                    except Exception:
                        pass
        paths.append(path_obj)
    return paths
# ----------------------------------------------------------------------------
# ZipSplat Model Initialization (loaded CPU-side, ZeroGPU moves to GPU)
# ----------------------------------------------------------------------------
model = ZipSplat(weights="zipsplat").eval()
@spaces.GPU
def generate(files: Any):
    paths = normalize_uploaded_paths(files)
    if not paths:
        default_placeholder = (
            '<div style="height: 500px; display: flex; align-items: center; justify-content: center; '
            'background: #080b12; color: #94a3b8; border-radius: 12px; font-family: system-ui, sans-serif;">'
            "Please upload images, videos, or Live Photo pairs.</div>"
        )
        return None, None, None, default_placeholder, "No files uploaded."
    try:
        result = load_media_views(paths, max_views=500, frames_per_video=8)
    except Exception as e:
        err_msg = f"Error loading media: {e}"
        err_placeholder = (
            f'<div style="height: 500px; display: flex; align-items: center; justify-content: center; '
            f'background: #080b12; color: #f87171; border-radius: 12px; font-family: system-ui, sans-serif;">'
            f"{err_msg}</div>"
        )
        return None, None, None, err_placeholder, err_msg
    if not result.images:
        status = "Error: No valid views could be loaded."
        if result.warnings:
            status += "\n\nWarnings:\n" + "\n".join(result.warnings)
        err_placeholder = (
            f'<div style="height: 500px; display: flex; align-items: center; justify-content: center; '
            f'background: #080b12; color: #f87171; border-radius: 12px; font-family: system-ui, sans-serif;">'
            f"{status}</div>"
        )
        return None, None, None, err_placeholder, status
    import traceback
    try:
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        model.to(device)
        gaussians = model(result.images)[0]
        video_path = tempfile.mktemp(suffix=".mp4")
        stroke_video_path = tempfile.mktemp(suffix="_stroke.mp4")
        ply_path = tempfile.mktemp(suffix=".ply")
        viz.turntable(gaussians, video_path, sweep_deg=None)
        render_stroke_turntable(gaussians, stroke_video_path)
        gaussians.save_ply(ply_path)
        center, radius = viz.scene_center_radius(gaussians)
        cx, cy, cz = center.tolist()
        viewer_html = build_viewer_iframe(ply_path, cx=cx, cy=cy, cz=cz, r=radius)
        total_g = gaussians.num_gaussians
        total_s = math.ceil(total_g / 32)
        status = (
            f"Loaded {result.image_count} still image(s) and selected "
            f"{result.selected_video_frames} of {result.decoded_video_frames} decoded video frame(s).\n"
            f"Total ZipSplat views: {len(result.images)}.\n"
            f"Generated 3D scene with {total_g:,} Gaussians across {total_s:,} strokes (32 Gaussians/stroke)."
        )
        if result.warnings:
            status += "\n\nWarnings:\n" + "\n".join(result.warnings)
        return video_path, stroke_video_path, ply_path, viewer_html, status
    except Exception as e:
        err_msg = f"Backend Crash during generation:\n{traceback.format_exc()}"
        err_html = (
            f'<div style="height: 500px; padding: 20px; overflow-y: auto; '
            f'background: #080b12; color: #f87171; border-radius: 12px; font-family: monospace;">'
            f"{err_msg.replace(chr(10), '<br>')}</div>"
        )
        return None, None, None, err_html, err_msg
# ----------------------------------------------------------------------------
# SEO Head Metadata & Social OpenGraph Tags
# ----------------------------------------------------------------------------
SEO_HEAD_HTML = """
<meta name="description" content="ZipSplatPlus: Instant single & multi-image 3D Gaussian Splatting with real-time stroke-by-stroke interactive 3D WebGL viewer on ZeroGPU. Convert photos, videos, and Apple Live Photos into 3D scenes!" />
<meta name="keywords" content="ZipSplat, ZipSplatPlus, 3D Gaussian Splatting, Stroke Playback, 3D Reconstruction, Single Image to 3D, Novel View Synthesis, Spark.js, ZeroGPU, HEIC 3D, Live Photo 3D, WebGL 3D Viewer" />
<meta name="author" content="ZipSplat Team" />
<meta name="robots" content="index, follow, max-image-preview:large" />
<link rel="canonical" href="https://huggingface.co/spaces/yosun/ZipSplatPlus" />
<!-- Open Graph / Facebook / LinkedIn / Slack / Discord -->
<meta property="og:type" content="website" />
<meta property="og:site_name" content="ZipSplatPlus" />
<meta property="og:title" content="ZipSplatPlus: Real-time 3D Gaussian Splatting & Stroke Playback" />
<meta property="og:description" content="Convert images, videos & Apple Live Photos to interactive 3D Gaussian Splatting scenes with stroke-by-stroke playback." />
<meta property="og:image" content="https://huggingface.co/spaces/yosun/ZipSplatPlus/resolve/main/thumbnail.png" />
<meta property="og:url" content="https://huggingface.co/spaces/yosun/ZipSplatPlus" />
<!-- Twitter / X Cards -->
<meta name="twitter:card" content="summary_large_image" />
<meta name="twitter:title" content="ZipSplatPlus: Real-time 3D Gaussian Splatting & Stroke Playback" />
<meta name="twitter:description" content="Instant 3D reconstruction with stroke-by-stroke playback powered by ZipSplat + Spark.js interactive viewer." />
<meta name="twitter:image" content="https://huggingface.co/spaces/yosun/ZipSplatPlus/resolve/main/thumbnail.png" />
<!-- Schema.org JSON-LD Microdata for Google Search Rich Results -->
<script type="application/ld+json">
{
  "@context": "https://schema.org",
  "@type": "WebApplication",
  "name": "ZipSplatPlus",
  "alternateName": "ZipSplat 3D Gaussian Splatting Demo",
  "url": "https://huggingface.co/spaces/yosun/ZipSplatPlus",
  "description": "State-of-the-art 3D Gaussian Splatting web application with interactive stroke-by-stroke 3D viewer, stroke video playback, 360° turntable renderer, and PLY exporter.",
  "applicationCategory": "MultimediaApplication",
  "operatingSystem": "All",
  "image": "https://huggingface.co/spaces/yosun/ZipSplatPlus/resolve/main/thumbnail.png",
  "offers": {
    "@type": "Offer",
    "price": "0",
    "priceCurrency": "USD"
  },
  "author": {
    "@type": "Organization",
    "name": "ZipSplat Team"
  }
}
</script>
"""
# ----------------------------------------------------------------------------
# Gradio UI Layout
# ----------------------------------------------------------------------------
with gr.Blocks(
    title="ZipSplatPlus: Real-time 3D Gaussian Splatting & Stroke Playback Viewer",
    head=SEO_HEAD_HTML,
) as demo:
    gr.Markdown("# 🔥 ZipSplatPlus: Real-time 3D Gaussian Splatting & Stroke Playback")
    gr.Markdown(
        "ZeroGPU Powered Demo for [veichta/zipsplat](https://huggingface.co/veichta/zipsplat). "
        "Upload images (JPEG, PNG, WebP, HEIC/HEIF), videos (MOV, MP4, M4V), or Apple Live Photo pairs "
        "(HEIC + MOV) to generate a 3D Gaussian Splatting scene and **play back each stroke step-by-step** in real time!"
    )
    with gr.Row():
        with gr.Column(scale=1):
            media_in = gr.File(
                label="Upload Images, Videos, or Live Photo Pairs",
                file_count="multiple",
                type="filepath",
                file_types=[
                    ".jpg",
                    ".jpeg",
                    ".png",
                    ".webp",
                    ".heic",
                    ".heif",
                    ".mov",
                    ".mp4",
                    ".m4v",
                ],
            )
            btn = gr.Button("🚀 Generate 3D Scene", variant="primary")
        with gr.Column(scale=2):
            viewer_out = gr.HTML(
                label="Interactive 3D Gaussian & Stroke Viewer (Spark.js)",
                value='<div style="height: 500px; display: flex; align-items: center; justify-content: center; background: #080b12; color: #94a3b8; border-radius: 12px; font-family: system-ui, sans-serif;">Upload media and click "Generate 3D Scene" to view interactive 3D Gaussian Splat with stroke-by-stroke playback</div>',
            )
            with gr.Tabs():
                with gr.TabItem("🖌️ Stroke-by-Stroke Playback Video"):
                    stroke_video_out = gr.Video(label="Stroke-by-Stroke Playback Video")
                with gr.TabItem("🎬 360° Turntable Video"):
                    video_out = gr.Video(label="360° Turntable Video Render")
            with gr.Row():
                file_out = gr.File(label="Download 3D Model (.ply)")
            status_out = gr.Textbox(label="Status & Warnings", interactive=False)
    btn.click(
        fn=generate,
        inputs=[media_in],
        outputs=[video_out, stroke_video_out, file_out, viewer_out, status_out],
        js="""function(files) {
            if (!files || (Array.isArray(files) && files.length === 0)) {
                alert("Please upload at least one image or video before generating.");
                throw new Error("No files uploaded");
            }
            return [files];
        }""",
    )
if __name__ == "__main__":
    demo.launch(allowed_paths=[tempfile.gettempdir()])