update space

.gitattributes

@@ -0,0 +1,37 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.ply filter=lfs diff=lfs merge=lfs -text
+*.whl filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+venv/
+weights/

README.md

@@ -0,0 +1,15 @@
+---
+title: VoMP
+emoji: 🚀
+colorFrom: green
+colorTo: green
+sdk: gradio
+python_version: 3.12
+sdk_version: 6.2.0
+app_file: app.py
+pinned: true
+license: apache-2.0
+short_description: Volumetric physics materials for interactive worlds
+suggested_hardware: a100-large
+suggested_storage: medium
+---

app.py

@@ -0,0 +1,466 @@
+import os
+import shutil
+import tempfile
+from typing import Any, Dict, List, Optional, Tuple
+
+import gradio as gr
+import matplotlib
+
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import matplotlib.colors as mcolors
+from matplotlib.colorbar import ColorbarBase
+import numpy as np
+import plotly.graph_objects as go
+import spaces
+import torch
+from huggingface_hub import snapshot_download
+
+from vomp.inference import Vomp
+from vomp.inference.utils import LazyLoadDino, save_materials
+
+NUM_VIEWS = 150
+PROPERTY_NAMES = ["youngs_modulus", "poissons_ratio", "density"]
+PROPERTY_DISPLAY_NAMES = {
+    "youngs_modulus": "Young's Modulus",
+    "poissons_ratio": "Poisson's Ratio",
+    "density": "Density",
+}
+
+BLENDER_LINK = (
+    "https://download.blender.org/release/Blender3.0/blender-3.0.1-linux-x64.tar.xz"
+)
+BLENDER_INSTALLATION_PATH = "/tmp"
+BLENDER_PATH = f"{BLENDER_INSTALLATION_PATH}/blender-3.0.1-linux-x64/blender"
+
+EXAMPLES_DIR = "examples"
+
+model_id = "nvidia/PhysicalAI-Simulation-VoMP-Model"
+base_path = snapshot_download(repo_id=model_id, local_dir="weights")
+print(os.listdir(base_path))
+
+
+def _preload_dino(model: Vomp) -> None:
+    print("Preloading DINO model...")
+    dino = LazyLoadDino(
+        model_name="dinov2_vitl14_reg",
+        device=model.device,
+        use_trt=getattr(model, "use_trt", False),
+    )
+    _ = dino.get_model()
+    _ = dino.get_transform()
+
+
+def _install_blender():
+    if not os.path.exists(BLENDER_PATH):
+        print("Installing Blender...")
+        os.system("sudo apt-get update")
+        os.system(
+            "sudo apt-get install -y libxrender1 libxi6 libxkbcommon-x11-0 libsm6"
+        )
+        os.system(f"wget {BLENDER_LINK} -P {BLENDER_INSTALLATION_PATH}")
+        os.system(
+            f"tar -xvf {BLENDER_INSTALLATION_PATH}/blender-3.0.1-linux-x64.tar.xz -C {BLENDER_INSTALLATION_PATH}"
+        )
+        print("Blender installed successfully!")
+
+
+def _is_gaussian_splat(file_path: str) -> bool:
+    if not file_path.lower().endswith(".ply"):
+        return False
+
+    try:
+        with open(file_path, "rb") as f:
+            header = b""
+            while True:
+                line = f.readline()
+                header += line
+                if b"end_header" in line:
+                    break
+                if len(header) > 10000:
+                    break
+
+            header_str = header.decode("utf-8", errors="ignore").lower()
+            gaussian_indicators = ["f_dc", "opacity", "scale_0", "rot_0"]
+            return any(indicator in header_str for indicator in gaussian_indicators)
+    except Exception:
+        return False
+
+
+def _setup_examples():
+    """Ensure examples directory exists."""
+    os.makedirs(EXAMPLES_DIR, exist_ok=True)
+
+
+_setup_examples()
+
+
+print("Loading VoMP model...")
+model = Vomp.from_checkpoint(
+    config_path="weights/inference.json",
+    geometry_checkpoint_dir="weights/geometry_transformer.pt",
+    matvae_checkpoint_dir="weights/matvae.safetensors",
+    normalization_params_path="weights/normalization_params.json",
+)
+print("VoMP model loaded successfully!")
+_preload_dino(model)
+
+
+def _create_colorbar(
+    data: np.ndarray, property_name: str, output_path: str, colormap: str = "viridis"
+) -> str:
+    fig, ax = plt.subplots(figsize=(6, 0.8))
+    fig.subplots_adjust(bottom=0.5)
+    ax.remove()
+
+    cmap = plt.cm.get_cmap(colormap)
+    norm = mcolors.Normalize(vmin=np.min(data), vmax=np.max(data))
+
+    cbar_ax = fig.add_axes([0.1, 0.4, 0.8, 0.35])
+    cb = ColorbarBase(cbar_ax, cmap=cmap, norm=norm, orientation="horizontal")
+    cb.ax.set_xlabel(
+        f"{PROPERTY_DISPLAY_NAMES.get(property_name, property_name)}", fontsize=10
+    )
+
+    plt.savefig(
+        output_path, dpi=150, bbox_inches="tight", facecolor="white", transparent=False
+    )
+    plt.close()
+    return output_path
+
+
+def _create_pointcloud_plot(
+    coords: np.ndarray,
+    values: np.ndarray,
+    property_name: str,
+    colormap: str = "viridis",
+) -> go.Figure:
+    """Create an interactive 3D colored point cloud plot."""
+    coords = np.asarray(coords, dtype=np.float32)
+    values = np.asarray(values, dtype=np.float32).reshape(-1)
+    if coords.ndim != 2 or coords.shape[1] != 3:
+        raise ValueError(f"coords must be (N,3), got {coords.shape}")
+    if values.shape[0] != coords.shape[0]:
+        raise ValueError(
+            f"values must be (N,), got {values.shape} for coords {coords.shape}"
+        )
+
+    fig = go.Figure(
+        data=[
+            go.Scatter3d(
+                x=coords[:, 0],
+                y=coords[:, 1],
+                z=coords[:, 2],
+                mode="markers",
+                marker={
+                    "size": 2,
+                    "color": values,
+                    "colorscale": colormap,
+                    "showscale": False,
+                    "opacity": 0.9,
+                },
+            )
+        ]
+    )
+    fig.update_layout(
+        title=f"{PROPERTY_DISPLAY_NAMES.get(property_name, property_name)} Point Cloud",
+        margin=dict(l=0, r=0, t=35, b=0),
+        scene=dict(
+            xaxis=dict(visible=False),
+            yaxis=dict(visible=False),
+            zaxis=dict(visible=False),
+            aspectmode="data",
+            bgcolor="rgb(20,20,20)",
+        ),
+        paper_bgcolor="rgb(20,20,20)",
+        plot_bgcolor="rgb(20,20,20)",
+        showlegend=False,
+    )
+    return fig
+
+
+def _create_material_visualizations(
+    material_file: str, output_dir: str
+) -> Dict[str, Tuple[Any, str]]:
+    result = {}
+    data = np.load(material_file, allow_pickle=True)
+
+    if "voxel_data" in data:
+        voxel_data = data["voxel_data"]
+        coords = np.column_stack([voxel_data["x"], voxel_data["y"], voxel_data["z"]])
+        properties = {
+            "youngs_modulus": voxel_data["youngs_modulus"],
+            "poissons_ratio": voxel_data["poissons_ratio"],
+            "density": voxel_data["density"],
+        }
+    else:
+        if "voxel_coords_world" in data:
+            coords = data["voxel_coords_world"]
+        elif "query_coords_world" in data:
+            coords = data["query_coords_world"]
+        elif "coords" in data:
+            coords = data["coords"]
+        else:
+            print(f"Warning: No coordinate data found in {material_file}")
+            return result
+
+        properties = {}
+        property_mapping = {
+            "youngs_modulus": ["youngs_modulus", "young_modulus"],
+            "poissons_ratio": ["poissons_ratio", "poisson_ratio"],
+            "density": ["density"],
+        }
+        for prop_name, possible_names in property_mapping.items():
+            for name in possible_names:
+                if name in data:
+                    properties[prop_name] = data[name]
+                    break
+
+    center = (np.min(coords, axis=0) + np.max(coords, axis=0)) / 2
+    max_range = np.max(np.max(coords, axis=0) - np.min(coords, axis=0))
+    if max_range > 1e-10:
+        coords_normalized = (coords - center) / max_range
+    else:
+        coords_normalized = coords - center
+
+    for prop_name, prop_data in properties.items():
+        if prop_data is not None:
+            plot = _create_pointcloud_plot(coords_normalized, prop_data, prop_name)
+            colorbar_path = os.path.join(output_dir, f"{prop_name}_colorbar.png")
+            _create_colorbar(prop_data, prop_name, colorbar_path)
+            result[prop_name] = (plot, colorbar_path)
+            print(f"Created point cloud plot for {prop_name}")
+
+    return result
+
+
+@spaces.GPU(duration=60)
+@torch.no_grad()
+def process_3d_model(input_file):
+    empty_result = (
+        None,  # youngs_cloud
+        None,  # youngs_colorbar
+        None,  # poissons_cloud
+        None,  # poissons_colorbar
+        None,  # density_cloud
+        None,  # density_colorbar
+        None,  # materials file
+    )
+
+    if input_file is None:
+        return empty_result
+
+    output_dir = tempfile.mkdtemp(prefix="vomp_")
+    material_file = os.path.join(output_dir, "materials.npz")
+
+    try:
+        if _is_gaussian_splat(input_file):
+            print(f"Processing as Gaussian splat: {input_file}")
+            results = model.get_splat_materials(
+                input_file,
+                voxel_method="kaolin",
+                query_points="voxel_centers",
+                output_dir=output_dir,
+            )
+        else:
+            print(f"Processing as mesh: {input_file}")
+            _install_blender()
+            results = model.get_mesh_materials(
+                input_file,
+                blender_path=BLENDER_PATH,
+                query_points="voxel_centers",
+                output_dir=output_dir,
+                return_original_scale=True,
+            )
+
+        save_materials(results, material_file)
+        print(f"Materials saved to: {material_file}")
+
+        visualizations = _create_material_visualizations(material_file, output_dir)
+
+        youngs_cloud = visualizations.get("youngs_modulus", (None, None))[0]
+        youngs_colorbar = visualizations.get("youngs_modulus", (None, None))[1]
+
+        poissons_cloud = visualizations.get("poissons_ratio", (None, None))[0]
+        poissons_colorbar = visualizations.get("poissons_ratio", (None, None))[1]
+
+        density_cloud = visualizations.get("density", (None, None))[0]
+        density_colorbar = visualizations.get("density", (None, None))[1]
+
+        return (
+            youngs_cloud,
+            youngs_colorbar,
+            poissons_cloud,
+            poissons_colorbar,
+            density_cloud,
+            density_colorbar,
+            material_file,
+        )
+
+    except Exception as e:
+        print(f"Error processing 3D model: {e}")
+        raise gr.Error(f"Failed to process 3D model: {str(e)}")
+
+
+css = """
+.gradio-container {
+    font-family: 'IBM Plex Sans', sans-serif;
+}
+
+.title-container {
+    text-align: center;
+    padding: 20px 0;
+}
+
+.badge-container {
+    display: flex;
+    justify-content: center;
+    gap: 8px;
+    flex-wrap: wrap;
+    margin-bottom: 20px;
+}
+
+.badge-container a img {
+    height: 22px;
+}
+
+h1 {
+    text-align: center;
+    font-size: 2.5rem;
+    margin-bottom: 0.5rem;
+}
+
+.subtitle {
+    text-align: center;
+    color: #666;
+    font-size: 1.1rem;
+    margin-bottom: 1.5rem;
+}
+
+.input-column, .output-column {
+    min-height: 400px;
+}
+
+.output-column .row {
+    display: flex !important;
+    flex-wrap: nowrap !important;
+    gap: 16px;
+}
+
+.output-column .row > .column {
+    flex: 1 1 50% !important;
+    min-width: 0 !important;
+}
+
+.main-content {
+    display: flex;
+    flex-direction: column-reverse;
+    gap: 16px;
+}
+"""
+
+title_md = """
+<div class="title-container">
+    <h1>VoMP: Predicting Volumetric Mechanical Properties</h1>
+    <p class="subtitle">Feed-forward, fine-grained, physically based volumetric material properties from Splats, Meshes, NeRFs, and more.</p>
+    <div class="badge-container">
+<a href="https://arxiv.org/abs/2510.22975"><img src='https://img.shields.io/badge/arXiv-VoMP-red' alt='Paper PDF'></a>
+<a href='https://research.nvidia.com/labs/sil/projects/vomp/'><img src='https://img.shields.io/badge/Project_Page-VoMP-green' alt='Project Page'></a>
+<a href='https://huggingface.co/nvidia/PhysicalAI-Simulation-VoMP-Model'><img src='https://img.shields.io/badge/%F0%9F%A4%97%20-Models-yellow'></a>
+<a href='https://huggingface.co/datasets/nvidia/PhysicalAI-Robotics-PhysicalAssets-VoMP'><img src='https://img.shields.io/badge/%F0%9F%A4%97%20-GVM%20Dataset-yellow'></a>
+    </div>
+</div>
+"""
+
+description_md = """
+Upload a Gaussian Splat (.ply) or Mesh (.obj, .glb, .stl, .gltf) to predict volumetric mechanical properties (Young's modulus, Poisson's ratio, density) for realistic physics simulation.
+
+### Tips for Best Results
+
+- Make sure the input asset has textures
+- Make sure the input asset is oriented y-up for best results
+- Download the `.npz` artifact to inspect the full-resolution outputs with the viewer we provide in the code release.
+"""
+
+
+with gr.Blocks(css=css, title="VoMP") as demo:
+    gr.HTML(title_md)
+    gr.Markdown(description_md)
+
+    with gr.Column(elem_classes="main-content"):
+        with gr.Row():
+            with gr.Column(scale=1, elem_classes="input-column"):
+                gr.Markdown("### 📤 Input")
+                input_model = gr.Model3D(
+                    label="Upload 3D Model",
+                    clear_color=[1.0, 1.0, 1.0, 1.0],
+                )
+
+                submit_btn = gr.Button(
+                    "🚀 Generate Materials", variant="primary", size="lg"
+                )
+
+            with gr.Column(scale=1, elem_classes="output-column"):
+                gr.Markdown("### 📥 Output - Material Properties")
+
+                # Row 1: Young's Modulus and Poisson's Ratio
+                with gr.Row():
+                    with gr.Column(scale=1, min_width=200):
+                        youngs_cloud = gr.Plot()
+                        youngs_colorbar = gr.Image(height=50, show_label=False)
+
+                    with gr.Column(scale=1, min_width=200):
+                        poissons_cloud = gr.Plot()
+                        poissons_colorbar = gr.Image(height=50, show_label=False)
+
+                # Row 2: Density and Download
+                with gr.Row():
+                    with gr.Column(scale=1, min_width=200):
+                        density_cloud = gr.Plot()
+                        density_colorbar = gr.Image(height=50, show_label=False)
+
+                    with gr.Column(scale=1, min_width=200):
+                        gr.Markdown("#### 💾 Download")
+                        output_file = gr.File(
+                            label="Download Materials (.npz)",
+                            file_count="single",
+                        )
+
+        gr.Examples(
+            examples=[
+                [os.path.join(EXAMPLES_DIR, "plant.ply")],
+                [os.path.join(EXAMPLES_DIR, "dog.ply")],
+                [os.path.join(EXAMPLES_DIR, "dozer.ply")],
+                [os.path.join(EXAMPLES_DIR, "fiscus.ply")],
+            ],
+            inputs=[input_model],
+            outputs=[
+                youngs_cloud,
+                youngs_colorbar,
+                poissons_cloud,
+                poissons_colorbar,
+                density_cloud,
+                density_colorbar,
+                output_file,
+            ],
+            fn=process_3d_model,
+            cache_examples=False,
+        )
+
+    # Event handlers
+    submit_btn.click(
+        fn=process_3d_model,
+        inputs=[input_model],
+        outputs=[
+            youngs_cloud,
+            youngs_colorbar,
+            poissons_cloud,
+            poissons_colorbar,
+            density_cloud,
+            density_colorbar,
+            output_file,
+        ],
+    )
+
+if __name__ == "__main__":
+    demo.launch()

examples/dog.ply

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1808d618588f67a083bca52848070297be63d24ba3ddf1e9dae7edd75d87c69e
+size 2477313

examples/dozer.ply

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b44ab835c41c4b4e2ad4b55456de1173f38e8bcef005de4d5d9d2c27f7b3749e
+size 84598795

examples/fiscus.ply

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7516d90f1ec0764ce653819057b1b69197a56bc31683b4a7fd4fdff247437f7e
+size 74925059

examples/plant.ply

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a659f85e78556a90cddb2f37fcf77e9bd452fda15de6f69f6db8c5803bf9fd5
+size 12734620

requirements.txt

@@ -0,0 +1,21 @@
+gradio
+polyscope==2.5.0
+trimesh==4.8.1
+Pillow==11.0.0
+safetensors==0.6.2
+easydict==1.13
+scipy==1.14.1
+pyparsing==3.2.3
+opencv-python-headless==4.10.0.84
+numpy==1.26.4
+matplotlib==3.7.5
+plotly
+torch==2.4.0
+torchvision==0.19.0
+xformers==0.0.27.post2
+spconv-cu121
+https://nvidia-kaolin.s3.us-east-2.amazonaws.com/torch-2.4.0_cu121/kaolin-0.18.0-cp312-cp312-linux_x86_64.whl
+https://huggingface.co/spaces/nvidia/PhysicalAI-Robotics-VoMP-Demo/resolve/main/wheels/utils3d-0.0.2-py3-none-any.whl
+https://huggingface.co/spaces/nvidia/PhysicalAI-Robotics-VoMP-Demo/resolve/main/wheels/diff_gaussian_rasterization-0.0.0-cp312-cp312-linux_x86_64.whl
+git+https://github.com/nv-tlabs/vomp.git#egg=vomp
+https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.0.post2/flash_attn-2.7.0.post2+cu12torch2.4cxx11abiFALSE-cp312-cp312-linux_x86_64.whl

wheels/diff_gaussian_rasterization-0.0.0-cp312-cp312-linux_x86_64.whl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b7166024db90f7878df26a50a5ec554a3a5efa421d2bc924a0fa1a99fb59b6c
+size 459917

wheels/utils3d-0.0.2-py3-none-any.whl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4caaf6a3e6ddca59d824d1436f6a6139d7410d61a5d33af5c5e57e041d1d21c2
+size 88958