"""SAM 3D Objects – pinned torch 2.8.0 for kaolin ABI compat.""" import os, sys, subprocess os.environ.setdefault("CUDA_HOME", "/usr/local/cuda") os.environ.setdefault("CONDA_PREFIX", "/usr/local") os.environ["LIDRA_SKIP_INIT"] = "true" import spaces import gradio as gr import numpy as np from PIL import Image from huggingface_hub import snapshot_download, login import tempfile, uuid from pathlib import Path if os.environ.get("HF_TOKEN"): login(token=os.environ["HF_TOKEN"]) # Runtime installs for things that need source builds def _pip(*a): r = subprocess.run([sys.executable, "-m", "pip", "install", "--no-cache-dir"] + list(a), capture_output=True, text=True, timeout=1200) return r.returncode == 0 _pip("utils3d") _pip("iopath") _pip("--no-deps", "pytorch3d") # gsplat for idx in ["https://docs.gsplat.studio/whl/pt28cu128", "https://docs.gsplat.studio/whl/pt27cu128", "https://docs.gsplat.studio/whl/pt26cu124"]: if _pip("--no-deps", f"--extra-index-url={idx}", "gsplat"): break _pip("--no-deps", "git+https://github.com/microsoft/MoGe.git@a8c37341bc0325ca99b9d57981cc3bb2bd3e255b") # Clone sam-3d-objects SAM3D_PATH = Path("/home/user/app/sam-3d-objects") if not SAM3D_PATH.exists(): subprocess.run(["git", "clone", "--depth", "1", "https://github.com/facebookresearch/sam-3d-objects.git", str(SAM3D_PATH)], check=True) subprocess.run([sys.executable, "-m", "pip", "install", "-e", str(SAM3D_PATH), "--no-deps"], capture_output=True, text=True) patch = SAM3D_PATH / "patching" / "hydra" if patch.exists(): subprocess.run(["bash", str(patch)], capture_output=True, cwd=str(SAM3D_PATH)) sys.path.insert(0, str(SAM3D_PATH)) sys.path.insert(0, str(SAM3D_PATH / "notebook")) # Pre-download checkpoints CKPT_DIR = snapshot_download(repo_id="facebook/sam-3d-objects", token=os.environ.get("HF_TOKEN")) hf_ckpt = Path(CKPT_DIR) / "checkpoints" local_ckpt = SAM3D_PATH / "checkpoints" / "hf" if hf_ckpt.exists() and not local_ckpt.exists(): local_ckpt.parent.mkdir(parents=True, exist_ok=True) local_ckpt.symlink_to(hf_ckpt) CONFIG_PATH = str(local_ckpt / "pipeline.yaml") print(f"Config exists: {Path(CONFIG_PATH).exists()}") # Verify for mod in ["torch", "kaolin", "gsplat", "open3d", "sam2"]: try: m = __import__(mod) print(f" {mod}={getattr(m, '__version__', 'ok')}") except Exception as e: print(f" {mod}: {e}") try: import kaolin; kaolin.ops.mesh print(" kaolin C++: OK") except Exception as e: print(f" kaolin C++: {e}") print("=== Setup done ===") SAM3D_MODEL = None SAM2_GEN = None @spaces.GPU(duration=60) def diagnose(): import torch lines = [f"torch={torch.__version__}", f"cuda={torch.cuda.is_available()}"] if torch.cuda.is_available(): lines.append(f"gpu={torch.cuda.get_device_name()}") for mod in ["kaolin", "gsplat", "open3d", "sam2", "utils3d"]: try: m = __import__(mod) lines.append(f"{mod}={getattr(m, '__version__', 'ok')}") except Exception as e: lines.append(f"{mod}: {e}") try: import kaolin; kaolin.ops.mesh lines.append("kaolin C++: OK") except Exception as e: lines.append(f"kaolin C++: {e}") return "\n".join(lines) @spaces.GPU(duration=300) def reconstruct_objects(image: np.ndarray): global SAM3D_MODEL, SAM2_GEN if image is None: return None, None, "No image" try: import torch, trimesh, time t0 = time.time() if SAM2_GEN is None: from sam2.automatic_mask_generator import SAM2AutomaticMaskGenerator SAM2_GEN = SAM2AutomaticMaskGenerator.from_pretrained("facebook/sam2-hiera-large") image_np = np.array(image) if not isinstance(image, np.ndarray) else image masks = SAM2_GEN.generate(image_np) if not masks: return None, image_np, "No objects" masks = sorted(masks, key=lambda x: x["area"], reverse=True) best_mask = masks[0]["segmentation"] preview = image_np.copy() preview[best_mask] = (preview[best_mask]*0.5 + np.array([0,255,0])*0.5).astype(np.uint8) print(f" SAM2: {len(masks)} masks ({time.time()-t0:.0f}s)") if SAM3D_MODEL is None: from inference import Inference SAM3D_MODEL = Inference(CONFIG_PATH, compile=False) print(f" SAM3D loaded ({time.time()-t0:.0f}s)") result = SAM3D_MODEL(image=image_np, mask=best_mask, seed=42) print(f" Reconstructed ({time.time()-t0:.0f}s)") if result is None: return None, preview, "Reconstruction None" od = tempfile.mkdtemp() glb = f"{od}/obj.glb" gs=None if hasattr(result,"save_ply"): gs=result elif isinstance(result,dict): for k in("gs","gaussian","gaussians"): v=result.get(k) if v: gs=v[0] if isinstance(v,(list,tuple)) else v; break if gs and hasattr(gs,"save_ply"): ply=f"{od}/t.ply"; gs.save_ply(ply) import open3d as o3d p=o3d.io.read_point_cloud(ply); p.estimate_normals() m,_=o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(p,depth=8) o3d.io.write_triangle_mesh(glb,m) elif gs and hasattr(gs,"_xyz"): import open3d as o3d p=o3d.geometry.PointCloud() p.points=o3d.utility.Vector3dVector(gs._xyz.detach().cpu().numpy()) p.estimate_normals() m,_=o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(p,depth=8) o3d.io.write_triangle_mesh(glb,m) else: return None,preview,f"No 3D: {type(result)}" n=0 try: n=len(trimesh.load(glb,force="mesh").faces) except: pass return glb,preview,f"OK: {n:,} faces ({int(time.time()-t0)}s)" except Exception as e: import traceback; traceback.print_exc() return None,None,f"Error: {e}" with gr.Blocks(title="SAM 3D Objects") as demo: gr.Markdown("# SAM 3D Objects\nImage -> 3D (GLB)") with gr.Tab("Reconstruct"): with gr.Row(): with gr.Column(): inp=gr.Image(label="Input",type="numpy") btn=gr.Button("Reconstruct",variant="primary",size="lg") with gr.Column(): prev=gr.Image(label="Detection",type="numpy",interactive=False) stat=gr.Textbox(label="Status") with gr.Row(): m3d=gr.Model3D(label="3D Preview") dl=gr.File(label="Download GLB") btn.click(reconstruct_objects,inputs=[inp],outputs=[m3d,prev,stat]) m3d.change(lambda x:x,inputs=[m3d],outputs=[dl]) with gr.Tab("Diagnose"): dbtn=gr.Button("GPU Diagnose") dout=gr.Textbox(label="Env",lines=15) dbtn.click(diagnose,outputs=[dout]) demo.launch(mcp_server=True)