import os import sys import subprocess import tempfile import spaces import torch import ctypes # Monkey-patch xformers.ops.memory_efficient_attention to fall back to # torch SDPA on Blackwell (sm_120). On the new ZeroGPU GPUs, neither # the FA3 nor the cutlass xformers kernel supports compute capability # 12.0, so any call into xformers' MEA raises NotImplementedError. # imagedream's mv_unet imports xformers.ops directly, so we have to # patch it before that import happens. import xformers import xformers.ops as _xops def _xformers_mea_sdpa(query, key, value, attn_bias=None, p=0.0, scale=None, op=None, **kwargs): # xformers MEA accepts (B, M, K) or (B, M, H, K). Torch SDPA wants # (B, H, M, K). Reshape appropriately and reverse on output. if query.dim() == 3: # Single-head: add an H=1 axis. q = query.unsqueeze(1) k = key.unsqueeze(1) v = value.unsqueeze(1) squeeze_out = True else: # (B, M, H, K) -> (B, H, M, K) q = query.transpose(1, 2) k = key.transpose(1, 2) v = value.transpose(1, 2) squeeze_out = False attn_mask = attn_bias if hasattr(attn_mask, "materialize"): try: attn_mask = attn_mask.materialize( shape=(q.shape[0], q.shape[1], q.shape[2], k.shape[2]), dtype=q.dtype, device=q.device, ) except Exception: attn_mask = None out = torch.nn.functional.scaled_dot_product_attention( q, k, v, attn_mask=attn_mask, dropout_p=p, scale=scale, ) if squeeze_out: return out.squeeze(1) return out.transpose(1, 2) _xops.memory_efficient_attention = _xformers_mea_sdpa xformers.ops.memory_efficient_attention = _xformers_mea_sdpa import gradio as gr import numpy as np from diffusers import DiffusionPipeline CUDA_HOME = "/cuda-image/usr/local/cuda-13.0" CUDA_LIBDIR = os.path.join(CUDA_HOME, "lib64") @spaces.GPU(duration=600) def _first_gpu_setup(): try: import diff_gaussian_rasterization # noqa return except ImportError: pass patch_dir = tempfile.mkdtemp(prefix="torch_cuda_patch_") with open(os.path.join(patch_dir, "sitecustomize.py"), "w") as f: f.write( "try:\n" " import torch.utils.cpp_extension as _c\n" " _c._check_cuda_version = lambda *a, **k: None\n" "except Exception:\n" " pass\n" ) env = os.environ.copy() env["CUDA_HOME"] = CUDA_HOME env["CUDA_PATH"] = CUDA_HOME env["PATH"] = os.path.join(CUDA_HOME, "bin") + os.pathsep + env.get("PATH", "") env["PYTHONPATH"] = patch_dir + os.pathsep + env.get("PYTHONPATH", "") env["TORCH_CUDA_ARCH_LIST"] = "12.0" subprocess.check_call( [sys.executable, "-m", "pip", "install", "--no-deps", "setuptools", "wheel", "ninja", "packaging"], ) # The PyPI release of `diff-gaussian-rasterization` is the original # graphdeco-inria version; the LGM model uses that one. subprocess.check_call( [sys.executable, "-m", "pip", "install", "--no-build-isolation", "--no-deps", "git+https://github.com/graphdeco-inria/diff-gaussian-rasterization.git"], env=env, ) _first_gpu_setup() try: ctypes.CDLL(os.path.join(CUDA_LIBDIR, "libcudart.so.13"), mode=ctypes.RTLD_GLOBAL) os.environ["LD_LIBRARY_PATH"] = CUDA_LIBDIR + os.pathsep + os.environ.get("LD_LIBRARY_PATH", "") except OSError: pass TMP_DIR = "/tmp" os.makedirs(TMP_DIR, exist_ok=True) image_pipeline = DiffusionPipeline.from_pretrained( "dylanebert/imagedream", custom_pipeline="dylanebert/multi-view-diffusion", torch_dtype=torch.float16, trust_remote_code=True, ).to("cuda") splat_pipeline = DiffusionPipeline.from_pretrained( "dylanebert/LGM", custom_pipeline="dylanebert/LGM", torch_dtype=torch.float16, trust_remote_code=True, ).to("cuda") @spaces.GPU def run(input_image, seed): np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) input_image = input_image.astype("float32") / 255.0 images = image_pipeline( "", input_image, guidance_scale=5, num_inference_steps=30, elevation=0 ) gaussians = splat_pipeline(images) output_ply_path = os.path.join(TMP_DIR, "output.ply") splat_pipeline.save_ply(gaussians, output_ply_path) return output_ply_path _TITLE = """LGM Mini""" _DESCRIPTION = """
A lightweight version of LGM: Large Multi-View Gaussian Model for High-Resolution 3D Content Creation. To convert to mesh, download the output splat and visit [splat-to-mesh](https://huggingface.co/spaces/dylanebert/splat-to-mesh).
""" css = """ #duplicate-button { margin: auto; color: white; background: #1565c0; border-radius: 100vh; } """ block = gr.Blocks(title=_TITLE, css=css) with block: gr.DuplicateButton( value="Duplicate Space for private use", elem_id="duplicate-button" ) with gr.Row(): with gr.Column(scale=1): gr.Markdown("# " + _TITLE) gr.Markdown(_DESCRIPTION) with gr.Row(variant="panel"): with gr.Column(scale=1): input_image = gr.Image(label="image", type="numpy") seed_input = gr.Number(label="seed", value=42) button_gen = gr.Button("Generate") with gr.Column(scale=1): output_splat = gr.Model3D(label="3D Gaussians") button_gen.click( fn=run, inputs=[input_image, seed_input], outputs=[output_splat] ) gr.Examples( examples=[ "https://huggingface.co/datasets/dylanebert/iso3d/resolve/main/jpg@512/a_cat_statue.jpg", "https://huggingface.co/datasets/dylanebert/iso3d/resolve/main/jpg@512/a_baby_penguin.jpg", "https://huggingface.co/datasets/dylanebert/iso3d/resolve/main/jpg@512/A_cartoon_house_with_red_roof.jpg", "https://huggingface.co/datasets/dylanebert/iso3d/resolve/main/jpg@512/a_hat.jpg", "https://huggingface.co/datasets/dylanebert/iso3d/resolve/main/jpg@512/an_antique_chest.jpg", "https://huggingface.co/datasets/dylanebert/iso3d/resolve/main/jpg@512/metal.jpg", ], inputs=[input_image], outputs=[output_splat], fn=lambda x: run(input_image=x, seed=42), cache_examples=True, label="Image-to-3D Examples", ) block.queue().launch(debug=True, share=True)