app.py

"""SAM 3D Objects – pinned torch 2.8.0 for kaolin ABI compat."""
import os, sys, subprocess
os.environ.setdefault("CUDA_HOME", "/usr/local/cuda")
os.environ.setdefault("CONDA_PREFIX", "/usr/local")
os.environ["LIDRA_SKIP_INIT"] = "true"

import spaces
import gradio as gr
import numpy as np
from PIL import Image
from huggingface_hub import snapshot_download, login
import tempfile, uuid
from pathlib import Path

if os.environ.get("HF_TOKEN"):
    login(token=os.environ["HF_TOKEN"])

# Runtime installs for things that need source builds
def _pip(*a):
    r = subprocess.run([sys.executable, "-m", "pip", "install", "--no-cache-dir"] + list(a),
                       capture_output=True, text=True, timeout=1200)
    return r.returncode == 0

_pip("utils3d")
_pip("iopath")
_pip("--no-deps", "pytorch3d")

# gsplat
for idx in ["https://docs.gsplat.studio/whl/pt28cu128",
            "https://docs.gsplat.studio/whl/pt27cu128",
            "https://docs.gsplat.studio/whl/pt26cu124"]:
    if _pip("--no-deps", f"--extra-index-url={idx}", "gsplat"):
        break

_pip("--no-deps", "git+https://github.com/microsoft/MoGe.git@a8c37341bc0325ca99b9d57981cc3bb2bd3e255b")

# Clone sam-3d-objects
SAM3D_PATH = Path("/home/user/app/sam-3d-objects")
if not SAM3D_PATH.exists():
    subprocess.run(["git", "clone", "--depth", "1",
        "https://github.com/facebookresearch/sam-3d-objects.git", str(SAM3D_PATH)], check=True)
subprocess.run([sys.executable, "-m", "pip", "install", "-e", str(SAM3D_PATH), "--no-deps"],
               capture_output=True, text=True)
patch = SAM3D_PATH / "patching" / "hydra"
if patch.exists():
    subprocess.run(["bash", str(patch)], capture_output=True, cwd=str(SAM3D_PATH))
sys.path.insert(0, str(SAM3D_PATH))
sys.path.insert(0, str(SAM3D_PATH / "notebook"))

# Pre-download checkpoints
CKPT_DIR = snapshot_download(repo_id="facebook/sam-3d-objects", token=os.environ.get("HF_TOKEN"))
hf_ckpt = Path(CKPT_DIR) / "checkpoints"
local_ckpt = SAM3D_PATH / "checkpoints" / "hf"
if hf_ckpt.exists() and not local_ckpt.exists():
    local_ckpt.parent.mkdir(parents=True, exist_ok=True)
    local_ckpt.symlink_to(hf_ckpt)
CONFIG_PATH = str(local_ckpt / "pipeline.yaml")
print(f"Config exists: {Path(CONFIG_PATH).exists()}")

# Verify
for mod in ["torch", "kaolin", "gsplat", "open3d", "sam2"]:
    try:
        m = __import__(mod)
        print(f"  {mod}={getattr(m, '__version__', 'ok')}")
    except Exception as e:
        print(f"  {mod}: {e}")
try:
    import kaolin; kaolin.ops.mesh
    print("  kaolin C++: OK")
except Exception as e:
    print(f"  kaolin C++: {e}")

print("=== Setup done ===")

SAM3D_MODEL = None
SAM2_GEN = None

@spaces.GPU(duration=60)
def diagnose():
    import torch
    lines = [f"torch={torch.__version__}", f"cuda={torch.cuda.is_available()}"]
    if torch.cuda.is_available():
        lines.append(f"gpu={torch.cuda.get_device_name()}")
    for mod in ["kaolin", "gsplat", "open3d", "sam2", "utils3d"]:
        try:
            m = __import__(mod)
            lines.append(f"{mod}={getattr(m, '__version__', 'ok')}")
        except Exception as e:
            lines.append(f"{mod}: {e}")
    try:
        import kaolin; kaolin.ops.mesh
        lines.append("kaolin C++: OK")
    except Exception as e:
        lines.append(f"kaolin C++: {e}")
    return "\n".join(lines)

@spaces.GPU(duration=300)
def reconstruct_objects(image: np.ndarray):
    global SAM3D_MODEL, SAM2_GEN
    if image is None:
        return None, None, "No image"
    try:
        import torch, trimesh, time
        t0 = time.time()

        if SAM2_GEN is None:
            from sam2.automatic_mask_generator import SAM2AutomaticMaskGenerator
            SAM2_GEN = SAM2AutomaticMaskGenerator.from_pretrained("facebook/sam2-hiera-large")
        image_np = np.array(image) if not isinstance(image, np.ndarray) else image
        masks = SAM2_GEN.generate(image_np)
        if not masks:
            return None, image_np, "No objects"
        masks = sorted(masks, key=lambda x: x["area"], reverse=True)
        best_mask = masks[0]["segmentation"]
        preview = image_np.copy()
        preview[best_mask] = (preview[best_mask]*0.5 + np.array([0,255,0])*0.5).astype(np.uint8)
        print(f"  SAM2: {len(masks)} masks ({time.time()-t0:.0f}s)")

        if SAM3D_MODEL is None:
            from inference import Inference
            SAM3D_MODEL = Inference(CONFIG_PATH, compile=False)
        print(f"  SAM3D loaded ({time.time()-t0:.0f}s)")

        result = SAM3D_MODEL(image=image_np, mask=best_mask, seed=42)
        print(f"  Reconstructed ({time.time()-t0:.0f}s)")

        if result is None:
            return None, preview, "Reconstruction None"

        od = tempfile.mkdtemp()
        glb = f"{od}/obj.glb"
        gs=None
        if hasattr(result,"save_ply"): gs=result
        elif isinstance(result,dict):
            for k in("gs","gaussian","gaussians"):
                v=result.get(k)
                if v: gs=v[0] if isinstance(v,(list,tuple)) else v; break
        if gs and hasattr(gs,"save_ply"):
            ply=f"{od}/t.ply"; gs.save_ply(ply)
            import open3d as o3d
            p=o3d.io.read_point_cloud(ply); p.estimate_normals()
            m,_=o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(p,depth=8)
            o3d.io.write_triangle_mesh(glb,m)
        elif gs and hasattr(gs,"_xyz"):
            import open3d as o3d
            p=o3d.geometry.PointCloud()
            p.points=o3d.utility.Vector3dVector(gs._xyz.detach().cpu().numpy())
            p.estimate_normals()
            m,_=o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(p,depth=8)
            o3d.io.write_triangle_mesh(glb,m)
        else:
            return None,preview,f"No 3D: {type(result)}"

        n=0
        try: n=len(trimesh.load(glb,force="mesh").faces)
        except: pass
        return glb,preview,f"OK: {n:,} faces ({int(time.time()-t0)}s)"
    except Exception as e:
        import traceback; traceback.print_exc()
        return None,None,f"Error: {e}"

with gr.Blocks(title="SAM 3D Objects") as demo:
    gr.Markdown("# SAM 3D Objects\nImage -> 3D (GLB)")
    with gr.Tab("Reconstruct"):
        with gr.Row():
            with gr.Column():
                inp=gr.Image(label="Input",type="numpy")
                btn=gr.Button("Reconstruct",variant="primary",size="lg")
            with gr.Column():
                prev=gr.Image(label="Detection",type="numpy",interactive=False)
                stat=gr.Textbox(label="Status")
        with gr.Row():
            m3d=gr.Model3D(label="3D Preview")
            dl=gr.File(label="Download GLB")
        btn.click(reconstruct_objects,inputs=[inp],outputs=[m3d,prev,stat])
        m3d.change(lambda x:x,inputs=[m3d],outputs=[dl])
    with gr.Tab("Diagnose"):
        dbtn=gr.Button("GPU Diagnose")
        dout=gr.Textbox(label="Env",lines=15)
        dbtn.click(diagnose,outputs=[dout])
demo.launch(mcp_server=True)