app.py

"""
SAM 3D Objects MCP Server
Image → 3D Object (GLB)

Automatic object detection with SAM2 + 3D reconstruction with SAM 3D Objects.
"""
import os
import sys
import subprocess
import tempfile
import uuid
from pathlib import Path

import gradio as gr
import numpy as np
import spaces
from huggingface_hub import snapshot_download, login
from PIL import Image

# Login with HF_TOKEN if available
if os.environ.get("HF_TOKEN"):
    login(token=os.environ.get("HF_TOKEN"))

# Clone sam-3d-objects repo if not exists
SAM3D_PATH = Path("/home/user/app/sam-3d-objects")
if not SAM3D_PATH.exists():
    print("Cloning sam-3d-objects repository...")
    subprocess.run([
        "git", "clone",
        "https://github.com/facebookresearch/sam-3d-objects.git",
        str(SAM3D_PATH)
    ], check=True)
    sys.path.insert(0, str(SAM3D_PATH))

sys.path.insert(0, str(SAM3D_PATH))

# Global models
SAM3D_MODEL = None
SAM2_GENERATOR = None


def load_sam2():
    """Load SAM2 automatic mask generator"""
    global SAM2_GENERATOR

    if SAM2_GENERATOR is not None:
        return SAM2_GENERATOR

    from sam2.automatic_mask_generator import SAM2AutomaticMaskGenerator

    print("Loading SAM2 model...")

    SAM2_GENERATOR = SAM2AutomaticMaskGenerator.from_pretrained("facebook/sam2-hiera-large")

    print("✓ SAM2 loaded")
    return SAM2_GENERATOR


def load_sam3d():
    """Load SAM 3D Objects model"""
    global SAM3D_MODEL

    if SAM3D_MODEL is not None:
        return SAM3D_MODEL

    import torch
    print("Loading SAM 3D Objects model...")

    checkpoint_dir = snapshot_download(
        repo_id="facebook/sam-3d-objects",
        token=os.environ.get("HF_TOKEN")
    )

    from sam_3d_objects import Sam3dObjects

    device = "cuda" if torch.cuda.is_available() else "cpu"
    SAM3D_MODEL = Sam3dObjects.from_pretrained(checkpoint_dir, device=device)

    print("✓ SAM 3D Objects loaded")
    return SAM3D_MODEL


@spaces.GPU(duration=120)
def reconstruct_objects(image: np.ndarray):
    """
    Automatically detect and reconstruct 3D objects from image.

    Args:
        image: Input RGB image

    Returns:
        tuple: (glb_path, preview_image, status)
    """
    if image is None:
        return None, None, "❌ No image provided"

    try:
        import torch
        import trimesh
        from PIL import Image as PILImage

        # Load models
        generator = load_sam2()
        sam3d = load_sam3d()

        # Convert to PIL if needed
        if isinstance(image, np.ndarray):
            pil_image = PILImage.fromarray(image)
        else:
            pil_image = image
            image = np.array(pil_image)

        # Auto-detect all objects
        print("Detecting objects...")
        masks = generator.generate(pil_image)

        if not masks or len(masks) == 0:
            return None, image, "⚠️ No objects detected"

        # Sort by area, take largest object
        masks = sorted(masks, key=lambda x: x['area'], reverse=True)
        best_mask = masks[0]['segmentation']

        # Create preview with mask overlay
        preview = image.copy()
        preview[best_mask] = (preview[best_mask] * 0.5 + np.array([0, 255, 0]) * 0.5).astype(np.uint8)

        # Run 3D reconstruction on largest object
        print("Reconstructing 3D...")
        mask_uint8 = best_mask.astype(np.uint8)
        outputs = sam3d.predict(image, mask_uint8)

        if outputs is None:
            return None, preview, "⚠️ 3D reconstruction failed"

        # Export as GLB
        output_dir = tempfile.mkdtemp()
        glb_path = f"{output_dir}/object_{uuid.uuid4().hex[:8]}.glb"

        # Get vertices from gaussian splat
        vertices = outputs.get_xyz().cpu().numpy()

        # Export as point cloud GLB
        cloud = trimesh.PointCloud(vertices)
        cloud.export(glb_path, file_type='glb')

        return glb_path, preview, f"✓ Detected {len(masks)} objects, reconstructed largest ({len(vertices)} points)"

    except Exception as e:
        import traceback
        traceback.print_exc()
        return None, None, f"❌ Error: {e}"


# Gradio Interface
with gr.Blocks(title="SAM 3D Objects MCP") as demo:
    gr.Markdown("""
    # 📦 SAM 3D Objects MCP Server
    **Image → 3D Object (GLB)**

    Automatically detects objects and reconstructs the largest one in 3D.
    """)

    with gr.Row():
        with gr.Column():
            input_image = gr.Image(label="Input Image", type="numpy")
            btn = gr.Button("🚀 Detect & Reconstruct", variant="primary", size="lg")

        with gr.Column():
            preview = gr.Image(label="Detected Object", type="numpy", interactive=False)
            status = gr.Textbox(label="Status")

    with gr.Row():
        with gr.Column():
            output_model = gr.Model3D(label="3D Preview")
        with gr.Column():
            output_file = gr.File(label="Download GLB")

    btn.click(
        reconstruct_objects,
        inputs=[input_image],
        outputs=[output_model, preview, status]
    )
    output_model.change(lambda x: x, inputs=[output_model], outputs=[output_file])

    gr.Markdown("""
    ---
    ### MCP Server
    ```json
    {
      "mcpServers": {
        "sam3d-objects": {
          "url": "https://dev-bjoern-sam3d-objects-mcp.hf.space/gradio_api/mcp/sse"
        }
      }
    }
    ```
    """)


if __name__ == "__main__":
    demo.launch(mcp_server=True)