import os os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import spaces # noqa: E402 (must be imported before torch) import math import random import tempfile import time import gradio as gr import numpy as np import torch import torch.nn.functional as F import trimesh import xatlas from diffusers import AutoencoderKL, ControlNetModel, EulerAncestralDiscreteScheduler, StableDiffusionXLControlNetPipeline from huggingface_hub import hf_hub_download from PIL import Image # --------------------------------------------------------------------------------------------- # Models: StableGen "Default" SDXL preset # checkpoint : RealVisXL V5.0 # LoRA : SDXL-Lightning 8-step (strength 1.0) # ControlNet : xinsir/controlnet-depth-sdxl-1.0 (strength 0.5) # sampler : 8 steps, CFG 1.5, dpmpp_2s_ancestral / sgm_uniform (-> Euler ancestral, trailing) # --------------------------------------------------------------------------------------------- DTYPE = torch.float16 controlnet = ControlNetModel.from_pretrained("xinsir/controlnet-depth-sdxl-1.0", torch_dtype=DTYPE) vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=DTYPE) pipe = StableDiffusionXLControlNetPipeline.from_pretrained( "SG161222/RealVisXL_V5.0", controlnet=controlnet, vae=vae, torch_dtype=DTYPE, variant="fp16" ) pipe.load_lora_weights(hf_hub_download("ByteDance/SDXL-Lightning", "sdxl_lightning_8step_lora.safetensors")) pipe.fuse_lora(lora_scale=1.0) pipe.unload_lora_weights() pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config, timestep_spacing="trailing") pipe.to("cuda") RES = 1024 # generation resolution per view (auto-rescale to ~1 MP) FOV = 2 * math.atan(18.0 / 50.0) # Blender default camera: 50 mm lens, 36 mm sensor MAX_FACES = 120_000 # --------------------------------------------------------------------------------------------- # Geometry helpers # --------------------------------------------------------------------------------------------- def load_mesh(path): scene = trimesh.load(path, force="scene") mesh = scene.to_geometry() if hasattr(scene, "to_geometry") else scene.dump(concatenate=True) mesh = trimesh.Trimesh(np.asarray(mesh.vertices), np.asarray(mesh.faces), process=True) if len(mesh.faces) == 0: raise gr.Error("The uploaded file contains no triangle mesh.") if len(mesh.faces) > MAX_FACES: mesh = mesh.simplify_quadric_decimation(face_count=MAX_FACES) return mesh def unwrap(mesh): vmap, faces, uvs = xatlas.parametrize( np.ascontiguousarray(mesh.vertices, dtype=np.float32), np.ascontiguousarray(mesh.faces, dtype=np.uint32) ) verts = np.asarray(mesh.vertices)[vmap] normals = np.asarray(mesh.vertex_normals)[vmap] return verts.astype(np.float32), normals.astype(np.float32), faces.astype(np.int32), uvs.astype(np.float32) def perspective(fov, near, far): f = 1.0 / math.tan(fov / 2) return np.array( [[f, 0, 0, 0], [0, f, 0, 0], [0, 0, (far + near) / (near - far), 2 * far * near / (near - far)], [0, 0, -1, 0]], dtype=np.float32, ) def look_at(eye, target=np.zeros(3), up=np.array([0.0, 1.0, 0.0])): fwd = target - eye fwd /= np.linalg.norm(fwd) if abs(np.dot(fwd, up)) > 0.999: up = np.array([0.0, 0.0, -1.0]) right = np.cross(fwd, up) right /= np.linalg.norm(right) true_up = np.cross(right, fwd) m = np.eye(4, dtype=np.float32) m[0, :3], m[1, :3], m[2, :3] = right, true_up, -fwd m[:3, 3] = -m[:3, :3] @ eye return m def camera_directions(num_cameras, elevation, top_view): """Orbit cameras, ordered front first and then alternating neighbours, so that every new view in the sequential workflow overlaps the ones generated before it.""" step = 360.0 / num_cameras order = [0] for k in range(1, num_cameras): order.append(((k + 1) // 2) * (1 if k % 2 else -1)) dirs = [] for o in order: az, el = math.radians(o * step), math.radians(elevation) dirs.append(np.array([math.sin(az) * math.cos(el), math.sin(el), math.cos(az) * math.cos(el)])) if top_view: el = math.radians(75) dirs.append(np.array([0.0, math.sin(el), math.cos(el)])) return dirs def fit_camera(verts, direction, margin=0.9): """Distance along `direction` at which the whole mesh fits the frame (like Blender's fit-to-view).""" eye0 = direction * 10.0 view = look_at(eye0) pc = verts @ view[:3, :3].T + view[:3, 3] z_toward = pc[:, 2] + 10.0 # coordinate along direction (towards the camera) t = math.tan(FOV / 2) * margin need = z_toward + np.maximum(np.abs(pc[:, 0]), np.abs(pc[:, 1])) / t return float(need.max()) def push_pull(col, w): """Fill pixels with w == 0 from their neighbours (pyramid push-pull).""" if min(col.shape[-2:]) <= 2: fill = (col * w).sum((-2, -1), keepdim=True) / w.sum((-2, -1), keepdim=True).clamp(min=1e-8) return col * w + fill * (1 - w) c = F.avg_pool2d(col * w, 2) ww = F.avg_pool2d(w, 2) small = push_pull(c / ww.clamp(min=1e-8), (ww > 0).float()) up = F.interpolate(small, size=col.shape[-2:], mode="bilinear", align_corners=False) return col * w + up * (1 - w) # --------------------------------------------------------------------------------------------- # Diffusion: SDXL + depth ControlNet with differential-diffusion inpainting (StableGen sequential) # --------------------------------------------------------------------------------------------- @torch.no_grad() def sample_view(prompt, negative_prompt, depth, context, regen_mask, steps, cfg, cn_scale, generator): """depth: (H,W) [0,1]; context: (H,W,3) [0,1] or None; regen_mask: (H,W) [0,1] (1 = regenerate).""" device = "cuda" do_cfg = cfg > 1.0 pe, npe, ppe, nppe = pipe.encode_prompt( prompt, device=device, num_images_per_prompt=1, do_classifier_free_guidance=do_cfg, negative_prompt=negative_prompt, ) time_ids = torch.tensor([[RES, RES, 0, 0, RES, RES]], device=device, dtype=pe.dtype) control = depth[None, None].expand(1, 3, RES, RES).to(device, DTYPE) if do_cfg: pe, ppe = torch.cat([npe, pe]), torch.cat([nppe, ppe]) time_ids = time_ids.repeat(2, 1) control = torch.cat([control, control]) cond = {"text_embeds": ppe, "time_ids": time_ids} sched = pipe.scheduler sched.set_timesteps(steps, device=device) noise = torch.randn((1, 4, RES // 8, RES // 8), generator=generator, device=device, dtype=torch.float32) latents = noise * sched.init_noise_sigma orig = m = None if context is not None: img = (context.permute(2, 0, 1)[None] * 2 - 1).to(device, DTYPE) orig = pipe.vae.encode(img).latent_dist.mean.float() * pipe.vae.config.scaling_factor m = F.interpolate(regen_mask[None, None].float(), size=(RES // 8, RES // 8), mode="area") for i, t in enumerate(sched.timesteps): if orig is not None: # Differential diffusion: a pixel with regenerate-strength s is only released for the # last s fraction of the schedule; before that it is pinned to the noised context. keep = m < (1.0 - i / steps) latents = torch.where(keep, sched.add_noise(orig, noise, t.reshape(1)), latents) lin = torch.cat([latents] * 2) if do_cfg else latents lin = sched.scale_model_input(lin, t).to(DTYPE) down, mid = pipe.controlnet( lin, t, encoder_hidden_states=pe, controlnet_cond=control, conditioning_scale=cn_scale, added_cond_kwargs=cond, return_dict=False, ) eps = pipe.unet( lin, t, encoder_hidden_states=pe, added_cond_kwargs=cond, down_block_additional_residuals=down, mid_block_additional_residual=mid, return_dict=False, )[0].float() if do_cfg: e_u, e_c = eps.chunk(2) eps = e_u + cfg * (e_c - e_u) latents = sched.step(eps, t, latents, generator=generator, return_dict=False)[0] out = pipe.vae.decode((latents / pipe.vae.config.scaling_factor).to(DTYPE), return_dict=False)[0] return ((out[0].float().permute(1, 2, 0) + 1) / 2).clamp(0, 1) def to_pil(t): return Image.fromarray((t.clamp(0, 1).cpu().numpy() * 255).astype(np.uint8)) def gaussian_blur(x, sigma=1.0, radius=1): k = torch.arange(-radius, radius + 1, device=x.device, dtype=x.dtype) k = torch.exp(-(k**2) / (2 * sigma**2)) k = k / k.sum() x = F.conv2d(F.pad(x, (radius, radius, 0, 0), mode="replicate"), k.view(1, 1, 1, -1)) return F.conv2d(F.pad(x, (0, 0, radius, radius), mode="replicate"), k.view(1, 1, -1, 1)) # --------------------------------------------------------------------------------------------- # GPU job: render guides, generate views sequentially, project + blend into a UV texture # --------------------------------------------------------------------------------------------- def _gpu_duration(verts, normals, faces, uvs, prompt, negative_prompt, num_cameras, elevation, top_view, seed, steps, guidance_scale, controlnet_scale, weight_exponent, texture_size, *args, **kwargs): n = int(num_cameras) + (1 if top_view else 0) # measured: 9 views x 8 steps @ 2048 tex = 19 s (incl. cold start) -> ~15% margin return int(min(120, 8 + n * (0.4 + 0.2 * int(steps)))) @spaces.GPU(duration=_gpu_duration) @torch.no_grad() def texture_mesh_gpu(verts, normals, faces, uvs, prompt, negative_prompt, num_cameras, elevation, top_view, seed, steps, guidance_scale, controlnet_scale, weight_exponent, texture_size, progress=gr.Progress(track_tqdm=True)): import nvdiffrast.torch as dr dev = "cuda" glctx = dr.RasterizeCudaContext() v = torch.from_numpy(verts).to(dev) nrm = F.normalize(torch.from_numpy(normals).to(dev), dim=-1) tri = torch.from_numpy(faces).to(dev) uv = torch.from_numpy(uvs).to(dev) T = int(texture_size) # --- texel buffers (rasterize in UV space) --- uv_clip = torch.cat([uv * 2 - 1, torch.zeros_like(uv[:, :1]), torch.ones_like(uv[:, :1])], -1)[None] rast_uv, _ = dr.rasterize(glctx, uv_clip.contiguous(), tri, resolution=[T, T]) tex_mask = (rast_uv[..., 3] > 0)[0] # (T,T), row 0 = v 0 tex_pos = dr.interpolate(v[None], rast_uv, tri)[0][0] tex_nrm = F.normalize(dr.interpolate(nrm[None], rast_uv, tri)[0][0], dim=-1) proj = torch.from_numpy(perspective(FOV, 0.05, 100.0)).to(dev) v_h = torch.cat([v, torch.ones_like(v[:, :1])], -1) p_h = torch.cat([tex_pos, torch.ones_like(tex_pos[..., :1])], -1) cams = [] for d in camera_directions(int(num_cameras), float(elevation), bool(top_view)): dist = fit_camera(verts, d) eye = d * dist cams.append((eye, torch.from_numpy(look_at(eye)).to(dev))) generator = torch.Generator(dev).manual_seed(int(seed)) colors, quals = [], [] # per camera: projected colour (T,T,3), visibility*|cos| (T,T) gallery = [] exp = float(weight_exponent) def blend(): if not colors: return None, None w = torch.stack(quals) ** exp col = (w[..., None] * torch.stack(colors)).sum(0) / w.sum(0)[..., None].clamp(min=1e-8) covered = (w.sum(0) > 0).float() return col, covered for ci, (eye, view) in enumerate(progress.tqdm(cams, desc="Generating views")): mvp = proj @ view clip = (v_h @ mvp.T)[None].contiguous() rast, _ = dr.rasterize(glctx, clip, tri, resolution=[RES, RES]) rast = rast.flip(1) # top row first fg = (rast[..., 3] > 0)[0] vz = -(v_h @ view.T)[:, 2:3] zbuf = dr.interpolate(vz[None].contiguous(), rast, tri)[0][0, ..., 0] zbuf = torch.where(fg, zbuf, torch.full_like(zbuf, 1e6)) # depth guide: normalize over the object, invert (near = white), background black zf = zbuf[fg] depth = torch.zeros((RES, RES), device=dev) depth[fg] = 1.0 - (zf - zf.min()) / (zf.max() - zf.min()).clamp(min=1e-6) context = regen = None if colors: col, covered = blend() ctx_tex = push_pull(col.permute(2, 0, 1)[None], covered[None, None])[0].permute(1, 2, 0) cov = 1 - torch.prod(1 - torch.stack(quals), 0) # angle-aware coverage by earlier views pix_uv = dr.interpolate(uv[None], rast, tri)[0][0] grid = (pix_uv * 2 - 1)[None] ctx = F.grid_sample(ctx_tex.permute(2, 0, 1)[None], grid, mode="bilinear", align_corners=False)[0] cov_px = F.grid_sample(cov[None, None], grid, mode="nearest", align_corners=False)[0, 0] context = torch.where(fg[..., None], ctx.permute(1, 2, 0), torch.full((RES, RES, 3), 0.5, device=dev)) # coverage ramp (low 0.15, high 1.0) -> keep; everything else is regenerated keep = ((cov_px - 0.15) / (1.0 - 0.15)).clamp(0, 1) * fg.float() regen = 1.0 - keep regen = F.max_pool2d(regen[None, None], 7, stride=1, padding=3) # grow mask by 3 px regen = gaussian_blur(regen, sigma=1.0, radius=1)[0, 0] image = sample_view(prompt, negative_prompt, depth, context, regen, int(steps), float(guidance_scale), float(controlnet_scale), generator) gallery.append((to_pil(image), f"View {ci + 1}")) gallery.append((to_pil(depth[..., None].expand(-1, -1, 3)), f"View {ci + 1} - depth guide")) # --- project this view onto the texels --- pc = p_h @ mvp.T w_clip = pc[..., 3] ndc = pc[..., :3] / w_clip[..., None].clamp(min=1e-6) inside = (ndc[..., 0].abs() <= 1) & (ndc[..., 1].abs() <= 1) & (w_clip > 0) & tex_mask grid = torch.stack([ndc[..., 0], -ndc[..., 1]], -1)[None] z_map = F.grid_sample(zbuf[None, None], grid, mode="nearest", align_corners=False)[0, 0] z_tex = -(p_h @ view.T)[..., 2] visible = inside & (z_tex <= z_map + 0.02) to_cam = F.normalize(torch.from_numpy(eye.astype(np.float32)).to(dev) - tex_pos, dim=-1) cosang = (tex_nrm * to_cam).sum(-1).abs() # backface projection allowed, discard angle 90 deg # soft silhouette guard: fade contributions from the outermost ~2 px of the generated object, # so background colour at slightly misaligned edges is not projected onto the surface fg_soft = -F.max_pool2d(-fg.float()[None, None], 5, stride=1, padding=2) fg_soft = gaussian_blur(fg_soft, sigma=1.0, radius=2) edge = F.grid_sample(fg_soft, grid, mode="bilinear", align_corners=False)[0, 0] quals.append(torch.where(visible, cosang * edge, torch.zeros_like(cosang))) col = F.grid_sample(image.permute(2, 0, 1)[None], grid, mode="bilinear", align_corners=False)[0] colors.append(col.permute(1, 2, 0)) col, covered = blend() final = push_pull(col.permute(2, 0, 1)[None], covered[None, None])[0].permute(1, 2, 0) tex_img = to_pil(final.flip(0)) # image rows top-first (v = 1 at the top) return tex_img, gallery # --------------------------------------------------------------------------------------------- # Gradio entry point # --------------------------------------------------------------------------------------------- def texture_mesh(mesh_file: str, prompt: str, negative_prompt: str = "", num_cameras: int = 4, elevation: float = 20.0, top_view: bool = True, seed: int = 0, randomize_seed: bool = True, steps: int = 8, guidance_scale: float = 1.5, controlnet_scale: float = 0.5, weight_exponent: float = 3.0, texture_size: int = 1024, progress=gr.Progress(track_tqdm=True)): """Texture a 3D mesh from a text prompt with StableGen's sequential SDXL projection workflow. Args: mesh_file: path to a .glb / .gltf / .obj / .ply / .stl mesh (existing materials are ignored). prompt: description of the surface, e.g. "weathered bronze statue, green patina". negative_prompt: optional negative prompt. num_cameras: number of orbit cameras around the object (4-8). elevation: elevation of the orbit cameras in degrees. top_view: add an extra camera looking down from above. seed: random seed. randomize_seed: pick a random seed instead of `seed`. steps: sampling steps per view (SDXL-Lightning 8-step LoRA). guidance_scale: classifier-free guidance scale. controlnet_scale: depth ControlNet strength. weight_exponent: view-weight exponent for blending projections (higher = sharper seams). texture_size: resolution of the baked UV texture. Returns: Textured GLB path, gallery of generated views and depth guides, and the used seed. """ if not mesh_file: raise gr.Error("Upload a mesh first.") if not prompt or not prompt.strip(): raise gr.Error("Enter a texture prompt.") if randomize_seed: seed = random.randint(0, 2**31 - 1) t0 = time.perf_counter() mesh = load_mesh(mesh_file) center = (mesh.bounds[0] + mesh.bounds[1]) / 2 scale = float(np.linalg.norm(mesh.vertices - center, axis=1).max()) norm_mesh = trimesh.Trimesh((mesh.vertices - center) / scale, mesh.faces, process=False) verts, normals, faces, uvs = unwrap(norm_mesh) print(f"[prep] {len(faces)} faces, unwrap {time.perf_counter() - t0:.1f}s") t1 = time.perf_counter() tex_img, gallery = texture_mesh_gpu(verts, normals, faces, uvs, prompt, negative_prompt, int(num_cameras), float(elevation), bool(top_view), int(seed), int(steps), float(guidance_scale), float(controlnet_scale), float(weight_exponent), int(texture_size)) print(f"[gpu] {time.perf_counter() - t1:.1f}s") material = trimesh.visual.material.PBRMaterial( baseColorTexture=tex_img, metallicFactor=0.0, roughnessFactor=1.0, doubleSided=True ) out = trimesh.Trimesh( vertices=verts * scale + center, faces=faces, visual=trimesh.visual.TextureVisuals(uv=uvs, material=material), process=False, ) path = tempfile.NamedTemporaryFile(suffix=".glb", delete=False).name out.export(path) return path, gallery, seed CSS = """ #col-container { max-width: 1200px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } """ with gr.Blocks() as demo: with gr.Column(elem_id="col-container"): gr.Markdown( """# StableGen: texture a 3D model from a prompt A port of [StableGen](https://github.com/sakalond/StableGen)'s core texturing workflow (SDXL **Default** preset, *Generate Sequentially*). Cameras are placed around the mesh; each view is generated with RealVisXL V5.0 + SDXL-Lightning, guided by a depth ControlNet, and inpaints only what earlier views have not covered. The views are projected back onto the surface, blended by viewing angle, and baked into a UV texture. [Wiki](https://github.com/sakalond/StableGen/wiki) ยท [Blender add-on](https://github.com/sakalond/StableGen/releases)""" ) with gr.Row(): with gr.Column(): mesh_in = gr.Model3D(label="Input mesh (.glb / .obj / .ply / .stl)", height=380) prompt = gr.Textbox(label="Texture prompt", placeholder="weathered bronze statue, green patina") run = gr.Button("Generate texture", variant="primary") with gr.Accordion("Advanced settings", open=False): negative_prompt = gr.Textbox(label="Negative prompt", value="") with gr.Row(): num_cameras = gr.Slider(4, 8, value=4, step=1, label="Number of orbit cameras") elevation = gr.Slider(-30, 60, value=20, step=1, label="Camera elevation (deg)") top_view = gr.Checkbox(value=True, label="Add top camera") with gr.Row(): seed = gr.Number(value=0, precision=0, label="Seed") randomize_seed = gr.Checkbox(value=True, label="Randomize seed") with gr.Row(): steps = gr.Slider(4, 12, value=8, step=1, label="Steps") guidance_scale = gr.Slider(1.0, 3.0, value=1.5, step=0.1, label="CFG") with gr.Row(): controlnet_scale = gr.Slider(0.0, 1.0, value=0.5, step=0.05, label="Depth ControlNet strength") weight_exponent = gr.Slider(1, 15, value=3, step=0.5, label="View weight exponent") texture_size = gr.Radio([1024, 2048], value=1024, label="Texture resolution") with gr.Column(): mesh_out = gr.Model3D(label="Textured mesh (GLB)", height=480) views = gr.Gallery(label="Generated views and depth guides", columns=4, height=300) inputs = [mesh_in, prompt, negative_prompt, num_cameras, elevation, top_view, seed, randomize_seed, steps, guidance_scale, controlnet_scale, weight_exponent, texture_size] run.click(texture_mesh, inputs=inputs, outputs=[mesh_out, views, seed], api_name="texture_mesh") gr.Examples( examples=[ ["examples/suzanne.glb", "weathered bronze statue, green patina"], ["examples/lantern.glb", "old rusty iron lantern with brass details, worn paint"], ["examples/barramundi_fish.glb", "a realistic barramundi fish, silver scales, golden fins"], ["examples/corset.glb", "victorian corset, dark red velvet with gold embroidery"], ], inputs=[mesh_in, prompt], outputs=[mesh_out, views, seed], fn=texture_mesh, cache_examples=True, cache_mode="lazy", ) demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)