update

README.md

@@ -4,7 +4,7 @@ emoji: 🎨
 colorFrom: yellow
 colorTo: pink
 sdk: gradio
-sdk_version: 5.33.1
+sdk_version: 6.8.0
 app_file: app.py
 pinned: false
 license: apache-2.0

app_style.py

@@ -20,7 +20,7 @@ from gradio.themes.utils.colors import gray, neutral, slate, stone, teal, zinc
 lighting_css = """
 <style>
 #lighter_mesh canvas {
-    filter: brightness(1) !important;
+    filter: brightness(2.3) !important;
 }
 </style>
 """

common.py

@@ -18,6 +18,11 @@ import spaces
 from embodied_gen.utils.monkey_patch.trellis import monkey_path_trellis
 
 monkey_path_trellis()
+from embodied_gen.utils.monkey_patch.gradio import (
+    _patch_open3d_cuda_device_count_bug,
+)
+
+_patch_open3d_cuda_device_count_bug()
 
 import gc
 import logging
@@ -73,7 +78,6 @@ current_file_path = os.path.abspath(__file__)
 current_dir = os.path.dirname(current_file_path)
 sys.path.append(os.path.join(current_dir, ".."))
 from thirdparty.TRELLIS.trellis.pipelines import TrellisImageTo3DPipeline
-from thirdparty.TRELLIS.trellis.utils import postprocessing_utils
 
 logging.basicConfig(
     format="%(asctime)s - %(levelname)s - %(message)s", level=logging.INFO
@@ -81,7 +85,7 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 os.environ["GRADIO_ANALYTICS_ENABLED"] = "false"
-
+os.environ.setdefault("OPENAI_API_KEY", "sk-placeholder")
 MAX_SEED = 100000
 
 # DELIGHT = DelightingModel()
@@ -92,7 +96,7 @@ if os.getenv("GRADIO_APP").startswith("imageto3d"):
     RBG14_REMOVER = BMGG14Remover()
     SAM_PREDICTOR = SAMPredictor(model_type="vit_h", device="cpu")
     if "sam3d" in os.getenv("GRADIO_APP"):
-        PIPELINE = Sam3dInference()
+        PIPELINE = Sam3dInference(device="cuda")
     else:
         PIPELINE = TrellisImageTo3DPipeline.from_pretrained(
             "microsoft/TRELLIS-image-large"
@@ -110,7 +114,7 @@ elif os.getenv("GRADIO_APP").startswith("textto3d"):
     RBG_REMOVER = RembgRemover()
     RBG14_REMOVER = BMGG14Remover()
     if "sam3d" in os.getenv("GRADIO_APP"):
-        PIPELINE = Sam3dInference()
+        PIPELINE = Sam3dInference(device="cuda")
     else:
         PIPELINE = TrellisImageTo3DPipeline.from_pretrained(
             "microsoft/TRELLIS-image-large"
@@ -157,7 +161,6 @@ def end_session(req: gr.Request) -> None:
         shutil.rmtree(user_dir)
 
 
-@spaces.GPU(duration=120)
 def preprocess_image_fn(
     image: str | np.ndarray | Image.Image,
     rmbg_tag: str = "rembg",
@@ -264,7 +267,7 @@ def select_point(
     return (image, masks), seg_image
 
 
-@spaces.GPU(duration=300)
+@spaces.GPU(duration=120)
 def image_to_3d(
     image: Image.Image,
     seed: int,
@@ -276,7 +279,7 @@ def image_to_3d(
     sam_image: Image.Image = None,
     is_sam_image: bool = False,
     req: gr.Request = None,
-) -> tuple[dict, str]:
+) -> tuple[object, str]:
     if is_sam_image:
         seg_image = filter_image_small_connected_components(sam_image)
         seg_image = Image.fromarray(seg_image, mode="RGBA")
@@ -286,6 +289,7 @@ def image_to_3d(
     if isinstance(seg_image, np.ndarray):
         seg_image = Image.fromarray(seg_image)
 
+    logger.info("Start generating 3D representation from image...")
     if isinstance(PIPELINE, Sam3dInference):
         outputs = PIPELINE.run(
             seg_image,
@@ -334,7 +338,7 @@ def image_to_3d(
 
 
 def extract_3d_representations_v2(
-    state: dict,
+    state: object,
     enable_delight: bool,
     texture_size: int,
     req: gr.Request,
@@ -401,7 +405,7 @@ def extract_3d_representations_v2(
 
 
 def extract_3d_representations_v3(
-    state: dict,
+    state: object,
     enable_delight: bool,
     texture_size: int,
     req: gr.Request,
@@ -564,7 +568,7 @@ def extract_urdf(
     )
 
 
-@spaces.GPU(duration=300)
+@spaces.GPU
 def text2image_fn(
     prompt: str,
     guidance_scale: float,
@@ -620,7 +624,7 @@ def text2image_fn(
     return save_paths + save_paths
 
 
-@spaces.GPU(duration=120)
+@spaces.GPU
 def generate_condition(mesh_path: str, req: gr.Request, uuid: str = "sample"):
     output_root = os.path.join(TMP_DIR, str(req.session_hash))
 
@@ -636,7 +640,7 @@ def generate_condition(mesh_path: str, req: gr.Request, uuid: str = "sample"):
     return None, None, None
 
 
-@spaces.GPU(duration=300)
+@spaces.GPU
 def generate_texture_mvimages(
     prompt: str,
     controlnet_cond_scale: float = 0.55,
@@ -723,7 +727,7 @@ def backproject_texture(
     return output_glb_mesh, output_obj_mesh, zip_file
 
 
-@spaces.GPU(duration=300)
+@spaces.GPU
 def backproject_texture_v2(
     mesh_path: str,
     input_image: str,
@@ -770,7 +774,7 @@ def backproject_texture_v2(
     return output_glb_mesh, output_obj_mesh, zip_file
 
 
-@spaces.GPU(duration=120)
+@spaces.GPU
 def render_result_video(
     mesh_path: str, video_size: int, req: gr.Request, uuid: str = ""
 ) -> str:

embodied_gen/data/backproject_v2.py

@@ -596,7 +596,7 @@ class TextureBacker:
 
         return texture
 
-    @spaces.GPU()
+    @spaces.GPU
     def compute_texture(
         self,
         colors: list[Image.Image],

embodied_gen/data/backproject_v3.py

@@ -425,7 +425,7 @@ def parse_args():
     return args
 
 
-@spaces.GPU()
+@spaces.GPU
 def entrypoint(
     delight_model: DelightingModel = None,
     imagesr_model: ImageRealESRGAN = None,

embodied_gen/data/mesh_operator.py

@@ -412,7 +412,7 @@ class MeshFixer(object):
             dtype=torch.int32,
         )
 
-    @spaces.GPU(duration=300)
+    @spaces.GPU
     def __call__(
         self,
         filter_ratio: float,

embodied_gen/models/delight_model.py

@@ -140,7 +140,7 @@ class DelightingModel(object):
 
         return new_image
 
-    @spaces.GPU(duration=120)
+    @spaces.GPU
     @torch.no_grad()
     def __call__(
         self,

embodied_gen/models/sam3d.py

@@ -51,6 +51,7 @@ class Sam3dInference:
     Args:
         local_dir (str): Directory to store or load model weights and configs.
         compile (bool): Whether to compile the model for faster inference.
+        device (str): Device to run the model on (e.g., "cuda" or "cpu").
 
     Methods:
         merge_mask_to_rgba(image, mask):
@@ -62,7 +63,10 @@ class Sam3dInference:
     """
 
     def __init__(
-        self, local_dir: str = "weights/sam-3d-objects", compile: bool = False
+        self,
+        local_dir: str = "weights/sam-3d-objects",
+        compile: bool = False,
+        device: str = "cuda",
     ) -> None:
         if not os.path.exists(local_dir):
             snapshot_download("facebook/sam-3d-objects", local_dir=local_dir)
@@ -78,6 +82,7 @@ class Sam3dInference:
         config["slat_decoder_gs_ckpt_path"] = config.pop(
             "slat_decoder_gs_4_ckpt_path", "slat_decoder_gs_4.ckpt"
         )
+        config["device"] = device
         self.pipeline: InferencePipelinePointMap = instantiate(config)
 
     def merge_mask_to_rgba(

embodied_gen/models/sr_model.py

@@ -80,7 +80,7 @@ class ImageStableSR:
         self.up_pipeline_x4.set_progress_bar_config(disable=True)
         # self.up_pipeline_x4.enable_model_cpu_offload()
 
-    @spaces.GPU(duration=120)
+    @spaces.GPU
     def __call__(
         self,
         image: Union[Image.Image, np.ndarray],
@@ -196,7 +196,7 @@ class ImageRealESRGAN:
                 half=True,
             )
 
-    @spaces.GPU(duration=120)
+    @spaces.GPU
     def __call__(self, image: Union[Image.Image, np.ndarray]) -> Image.Image:
         """Performs super-resolution on the input image.
 

embodied_gen/scripts/render_gs.py

@@ -96,7 +96,7 @@ def parse_args():
     return args
 
 
-@spaces.GPU(duration=120)
+@spaces.GPU
 def entrypoint(**kwargs) -> None:
     args = parse_args()
     for k, v in kwargs.items():

embodied_gen/utils/monkey_patch/gradio.py

@@ -0,0 +1,58 @@
+# Project EmbodiedGen
+#
+# Copyright (c) 2025 Horizon Robotics. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+
+import fileinput
+import site
+
+import gradio_client.utils as gradio_client_utils
+
+
+def _patch_gradio_schema_bool_bug() -> None:
+    """Patch schema parser for bool-style for gradio<5.33."""
+    original_get_type = gradio_client_utils.get_type
+    original_json_schema_to_python_type = (
+        gradio_client_utils._json_schema_to_python_type
+    )
+
+    def _safe_get_type(schema):
+        if isinstance(schema, bool):
+            return {}
+        return original_get_type(schema)
+
+    def _safe_json_schema_to_python_type(schema, defs):
+        if isinstance(schema, bool):
+            return "Any"
+        return original_json_schema_to_python_type(schema, defs)
+
+    gradio_client_utils.get_type = _safe_get_type
+    gradio_client_utils._json_schema_to_python_type = (
+        _safe_json_schema_to_python_type
+    )
+
+
+def _patch_open3d_cuda_device_count_bug() -> None:
+    """Patch open3d to avoid cuda device count bug."""
+    with fileinput.FileInput(
+        f'{site.getsitepackages()[0]}/open3d/__init__.py', inplace=True
+    ) as file:
+        for line in file:
+            print(
+                line.replace(
+                    '_pybind_cuda.open3d_core_cuda_device_count()', '1'
+                ),
+                end='',
+            )

embodied_gen/utils/monkey_patch/sam3d.py

@@ -30,6 +30,7 @@ def monkey_patch_sam3d():
     from embodied_gen.utils.log import logger
 
     os.environ["LIDRA_SKIP_INIT"] = "true"
+    os.environ['ATTN_BACKEND'] = "xformers"
 
     current_file_path = os.path.abspath(__file__)
     current_dir = os.path.dirname(current_file_path)
@@ -379,7 +380,7 @@ def monkey_patch_sam3d():
 
         InferencePipeline.__init__ = patch_init
 
-    patch_pointmap_infer_pipeline()
-    patch_infer_init()
+    patch_pointmap_infer_pipeline()  # patch
+    patch_infer_init()  # patch
 
     return

embodied_gen/utils/monkey_patch/trellis.py

@@ -37,7 +37,7 @@ def monkey_path_trellis():
     os.environ["TORCH_EXTENSIONS_DIR"] = os.path.expanduser(
         "~/.cache/torch_extensions"
     )
-    os.environ["SPCONV_ALGO"] = "auto"  # Can be 'native' or 'auto'
+    os.environ["SPCONV_ALGO"] = "native"  # Can be 'native' or 'auto'
     os.environ['ATTN_BACKEND'] = (
         "xformers"  # Can be 'flash-attn' or 'xformers'
     )

embodied_gen/utils/process_media.py

@@ -53,7 +53,7 @@ __all__ = [
 ]
 
 
-@spaces.GPU(duration=120)
+@spaces.GPU
 def render_asset3d(
     mesh_path: str,
     output_root: str,

embodied_gen/utils/trender.py

@@ -19,7 +19,6 @@ import sys
 from collections import defaultdict
 
 import numpy as np
-import spaces
 import torch
 from easydict import EasyDict as edict
 from tqdm import tqdm
@@ -43,7 +42,6 @@ __all__ = [
 ]
 
 
-@spaces.GPU(duration=120)
 def render_mesh_frames(sample, extrinsics, intrinsics, options={}, **kwargs):
     renderer = MeshRenderer()
     renderer.rendering_options.resolution = options.get("resolution", 512)
@@ -66,7 +64,6 @@ def render_mesh_frames(sample, extrinsics, intrinsics, options={}, **kwargs):
     return rets
 
 
-@spaces.GPU(duration=120)
 def render_gs_frames(
     sample,
     extrinsics,
@@ -117,7 +114,6 @@ def render_gs_frames(
     return dict(outputs)
 
 
-@spaces.GPU(duration=120)
 def render_video(
     sample,
     resolution=512,
@@ -149,7 +145,6 @@ def render_video(
     return result
 
 
-@spaces.GPU(duration=120)
 def pack_state(gs: Gaussian, mesh: MeshExtractResult) -> dict:
     return {
         "gaussian": {

requirements.txt

@@ -20,7 +20,8 @@ igraph==0.11.8
 pyvista==0.36.1
 openai==1.58.1
 transformers==4.42.4
-gradio[oauth,mcp]==5.33.1
+gradio==6.8.0
+spaces==0.47.0
 sentencepiece==0.2.0
 diffusers==0.31.0
 xatlas==0.0.9
@@ -31,18 +32,10 @@ basicsr==1.4.2
 realesrgan==0.3.0
 pydantic
 vtk==9.3.1
-spaces
 # utils3d@git+https://github.com/EasternJournalist/utils3d.git@9a4eb15e4021b67b12c460c7057d642626897ec8
 clip@git+https://github.com/openai/CLIP.git
 kolors@git+https://github.com/Kwai-Kolors/Kolors.git#egg=038818d
 segment-anything@git+https://github.com/facebookresearch/segment-anything.git#egg=dca509f
-https://github.com/nerfstudio-project/gsplat/releases/download/v1.5.0/gsplat-1.5.0+pt24cu121-cp310-cp310-linux_x86_64.whl
-https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.0.post2/flash_attn-2.7.0.post2+cu12torch2.4cxx11abiFALSE-cp310-cp310-linux_x86_64.whl
-# kaolin@git+https://github.com/NVIDIAGameWorks/kaolin.git@v0.16.0
-https://huggingface.co/xinjjj/RoboAssetGen/resolve/main/wheel_cu121/kaolin-0.16.0-cp310-cp310-linux_x86_64.whl
-# nvdiffrast@git+https://github.com/NVlabs/nvdiffrast.git#egg=729261d
-https://huggingface.co/xinjjj/RoboAssetGen/resolve/main/wheel_cu121/nvdiffrast-0.3.3-cp310-cp310-linux_x86_64.whl
-https://huggingface.co/xinjjj/RoboAssetGen/resolve/main/wheel_cu121/diff_gaussian_rasterization-0.0.0-cp310-cp310-linux_x86_64.whl
 colorlog
 json-repair
 scikit-learn
@@ -62,8 +55,19 @@ loguru
 seaborn
 hydra-core
 modelscope
-# git+https://github.com/facebookresearch/pytorch3d.git@stable
-https://huggingface.co/xinjjj/RoboAssetGen/resolve/main/wheel_cu121/pytorch3d-0.7.8-cp310-cp310-linux_x86_64.whl
 timm
-open3d
-MoGe@git+https://github.com/microsoft/MoGe.git@a8c3734
+# open3d
+# MoGe@git+https://github.com/microsoft/MoGe.git@a8c3734
+
+
+# git+https://github.com/facebookresearch/pytorch3d.git@stable
+# https://huggingface.co/xinjjj/RoboAssetGen/resolve/main/wheel_cu121/pytorch3d-0.7.8-cp310-cp310-linux_x86_64.whl
+# git+https://github.com/nerfstudio-project/gsplat.git@v1.5.3
+https://github.com/nerfstudio-project/gsplat/releases/download/v1.5.0/gsplat-1.5.0+pt24cu121-cp310-cp310-linux_x86_64.whl
+# flash-attn==2.7.0.post2
+# https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.0.post2/flash_attn-2.7.0.post2+cu12torch2.4cxx11abiFALSE-cp310-cp310-linux_x86_64.whl
+# kaolin@git+https://github.com/NVIDIAGameWorks/kaolin.git@v0.16.0
+https://huggingface.co/xinjjj/RoboAssetGen/resolve/main/wheel_cu121/kaolin-0.16.0-cp310-cp310-linux_x86_64.whl
+# nvdiffrast@git+https://github.com/NVlabs/nvdiffrast.git#egg=729261d
+https://huggingface.co/xinjjj/RoboAssetGen/resolve/main/wheel_cu121/nvdiffrast-0.3.3-cp310-cp310-linux_x86_64.whl
+https://huggingface.co/xinjjj/RoboAssetGen/resolve/main/wheel_cu121/diff_gaussian_rasterization-0.0.0-cp310-cp310-linux_x86_64.whl