stablegen / app.py
multimodalart's picture
multimodalart HF Staff
tighten GPU duration
d5b702b verified
Raw History Blame Contribute Delete
22.1 kB
import os
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import spaces # noqa: E402 (must be imported before torch)
import math
import random
import tempfile
import time
import gradio as gr
import numpy as np
import torch
import torch.nn.functional as F
import trimesh
import xatlas
from diffusers import AutoencoderKL, ControlNetModel, EulerAncestralDiscreteScheduler, StableDiffusionXLControlNetPipeline
from huggingface_hub import hf_hub_download
from PIL import Image
# ---------------------------------------------------------------------------------------------
# Models: StableGen "Default" SDXL preset
# checkpoint : RealVisXL V5.0
# LoRA : SDXL-Lightning 8-step (strength 1.0)
# ControlNet : xinsir/controlnet-depth-sdxl-1.0 (strength 0.5)
# sampler : 8 steps, CFG 1.5, dpmpp_2s_ancestral / sgm_uniform (-> Euler ancestral, trailing)
# ---------------------------------------------------------------------------------------------
DTYPE = torch.float16
controlnet = ControlNetModel.from_pretrained("xinsir/controlnet-depth-sdxl-1.0", torch_dtype=DTYPE)
vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=DTYPE)
pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
"SG161222/RealVisXL_V5.0", controlnet=controlnet, vae=vae, torch_dtype=DTYPE, variant="fp16"
)
pipe.load_lora_weights(hf_hub_download("ByteDance/SDXL-Lightning", "sdxl_lightning_8step_lora.safetensors"))
pipe.fuse_lora(lora_scale=1.0)
pipe.unload_lora_weights()
pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config, timestep_spacing="trailing")
pipe.to("cuda")
RES = 1024 # generation resolution per view (auto-rescale to ~1 MP)
FOV = 2 * math.atan(18.0 / 50.0) # Blender default camera: 50 mm lens, 36 mm sensor
MAX_FACES = 120_000
# ---------------------------------------------------------------------------------------------
# Geometry helpers
# ---------------------------------------------------------------------------------------------
def load_mesh(path):
scene = trimesh.load(path, force="scene")
mesh = scene.to_geometry() if hasattr(scene, "to_geometry") else scene.dump(concatenate=True)
mesh = trimesh.Trimesh(np.asarray(mesh.vertices), np.asarray(mesh.faces), process=True)
if len(mesh.faces) == 0:
raise gr.Error("The uploaded file contains no triangle mesh.")
if len(mesh.faces) > MAX_FACES:
mesh = mesh.simplify_quadric_decimation(face_count=MAX_FACES)
return mesh
def unwrap(mesh):
vmap, faces, uvs = xatlas.parametrize(
np.ascontiguousarray(mesh.vertices, dtype=np.float32), np.ascontiguousarray(mesh.faces, dtype=np.uint32)
)
verts = np.asarray(mesh.vertices)[vmap]
normals = np.asarray(mesh.vertex_normals)[vmap]
return verts.astype(np.float32), normals.astype(np.float32), faces.astype(np.int32), uvs.astype(np.float32)
def perspective(fov, near, far):
f = 1.0 / math.tan(fov / 2)
return np.array(
[[f, 0, 0, 0], [0, f, 0, 0], [0, 0, (far + near) / (near - far), 2 * far * near / (near - far)], [0, 0, -1, 0]],
dtype=np.float32,
)
def look_at(eye, target=np.zeros(3), up=np.array([0.0, 1.0, 0.0])):
fwd = target - eye
fwd /= np.linalg.norm(fwd)
if abs(np.dot(fwd, up)) > 0.999:
up = np.array([0.0, 0.0, -1.0])
right = np.cross(fwd, up)
right /= np.linalg.norm(right)
true_up = np.cross(right, fwd)
m = np.eye(4, dtype=np.float32)
m[0, :3], m[1, :3], m[2, :3] = right, true_up, -fwd
m[:3, 3] = -m[:3, :3] @ eye
return m
def camera_directions(num_cameras, elevation, top_view):
"""Orbit cameras, ordered front first and then alternating neighbours, so that every new
view in the sequential workflow overlaps the ones generated before it."""
step = 360.0 / num_cameras
order = [0]
for k in range(1, num_cameras):
order.append(((k + 1) // 2) * (1 if k % 2 else -1))
dirs = []
for o in order:
az, el = math.radians(o * step), math.radians(elevation)
dirs.append(np.array([math.sin(az) * math.cos(el), math.sin(el), math.cos(az) * math.cos(el)]))
if top_view:
el = math.radians(75)
dirs.append(np.array([0.0, math.sin(el), math.cos(el)]))
return dirs
def fit_camera(verts, direction, margin=0.9):
"""Distance along `direction` at which the whole mesh fits the frame (like Blender's fit-to-view)."""
eye0 = direction * 10.0
view = look_at(eye0)
pc = verts @ view[:3, :3].T + view[:3, 3]
z_toward = pc[:, 2] + 10.0 # coordinate along direction (towards the camera)
t = math.tan(FOV / 2) * margin
need = z_toward + np.maximum(np.abs(pc[:, 0]), np.abs(pc[:, 1])) / t
return float(need.max())
def push_pull(col, w):
"""Fill pixels with w == 0 from their neighbours (pyramid push-pull)."""
if min(col.shape[-2:]) <= 2:
fill = (col * w).sum((-2, -1), keepdim=True) / w.sum((-2, -1), keepdim=True).clamp(min=1e-8)
return col * w + fill * (1 - w)
c = F.avg_pool2d(col * w, 2)
ww = F.avg_pool2d(w, 2)
small = push_pull(c / ww.clamp(min=1e-8), (ww > 0).float())
up = F.interpolate(small, size=col.shape[-2:], mode="bilinear", align_corners=False)
return col * w + up * (1 - w)
# ---------------------------------------------------------------------------------------------
# Diffusion: SDXL + depth ControlNet with differential-diffusion inpainting (StableGen sequential)
# ---------------------------------------------------------------------------------------------
@torch.no_grad()
def sample_view(prompt, negative_prompt, depth, context, regen_mask, steps, cfg, cn_scale, generator):
"""depth: (H,W) [0,1]; context: (H,W,3) [0,1] or None; regen_mask: (H,W) [0,1] (1 = regenerate)."""
device = "cuda"
do_cfg = cfg > 1.0
pe, npe, ppe, nppe = pipe.encode_prompt(
prompt, device=device, num_images_per_prompt=1, do_classifier_free_guidance=do_cfg,
negative_prompt=negative_prompt,
)
time_ids = torch.tensor([[RES, RES, 0, 0, RES, RES]], device=device, dtype=pe.dtype)
control = depth[None, None].expand(1, 3, RES, RES).to(device, DTYPE)
if do_cfg:
pe, ppe = torch.cat([npe, pe]), torch.cat([nppe, ppe])
time_ids = time_ids.repeat(2, 1)
control = torch.cat([control, control])
cond = {"text_embeds": ppe, "time_ids": time_ids}
sched = pipe.scheduler
sched.set_timesteps(steps, device=device)
noise = torch.randn((1, 4, RES // 8, RES // 8), generator=generator, device=device, dtype=torch.float32)
latents = noise * sched.init_noise_sigma
orig = m = None
if context is not None:
img = (context.permute(2, 0, 1)[None] * 2 - 1).to(device, DTYPE)
orig = pipe.vae.encode(img).latent_dist.mean.float() * pipe.vae.config.scaling_factor
m = F.interpolate(regen_mask[None, None].float(), size=(RES // 8, RES // 8), mode="area")
for i, t in enumerate(sched.timesteps):
if orig is not None:
# Differential diffusion: a pixel with regenerate-strength s is only released for the
# last s fraction of the schedule; before that it is pinned to the noised context.
keep = m < (1.0 - i / steps)
latents = torch.where(keep, sched.add_noise(orig, noise, t.reshape(1)), latents)
lin = torch.cat([latents] * 2) if do_cfg else latents
lin = sched.scale_model_input(lin, t).to(DTYPE)
down, mid = pipe.controlnet(
lin, t, encoder_hidden_states=pe, controlnet_cond=control, conditioning_scale=cn_scale,
added_cond_kwargs=cond, return_dict=False,
)
eps = pipe.unet(
lin, t, encoder_hidden_states=pe, added_cond_kwargs=cond,
down_block_additional_residuals=down, mid_block_additional_residual=mid, return_dict=False,
)[0].float()
if do_cfg:
e_u, e_c = eps.chunk(2)
eps = e_u + cfg * (e_c - e_u)
latents = sched.step(eps, t, latents, generator=generator, return_dict=False)[0]
out = pipe.vae.decode((latents / pipe.vae.config.scaling_factor).to(DTYPE), return_dict=False)[0]
return ((out[0].float().permute(1, 2, 0) + 1) / 2).clamp(0, 1)
def to_pil(t):
return Image.fromarray((t.clamp(0, 1).cpu().numpy() * 255).astype(np.uint8))
def gaussian_blur(x, sigma=1.0, radius=1):
k = torch.arange(-radius, radius + 1, device=x.device, dtype=x.dtype)
k = torch.exp(-(k**2) / (2 * sigma**2))
k = k / k.sum()
x = F.conv2d(F.pad(x, (radius, radius, 0, 0), mode="replicate"), k.view(1, 1, 1, -1))
return F.conv2d(F.pad(x, (0, 0, radius, radius), mode="replicate"), k.view(1, 1, -1, 1))
# ---------------------------------------------------------------------------------------------
# GPU job: render guides, generate views sequentially, project + blend into a UV texture
# ---------------------------------------------------------------------------------------------
def _gpu_duration(verts, normals, faces, uvs, prompt, negative_prompt, num_cameras, elevation, top_view,
seed, steps, guidance_scale, controlnet_scale, weight_exponent, texture_size, *args, **kwargs):
n = int(num_cameras) + (1 if top_view else 0)
# measured: 9 views x 8 steps @ 2048 tex = 19 s (incl. cold start) -> ~15% margin
return int(min(120, 8 + n * (0.4 + 0.2 * int(steps))))
@spaces.GPU(duration=_gpu_duration)
@torch.no_grad()
def texture_mesh_gpu(verts, normals, faces, uvs, prompt, negative_prompt, num_cameras, elevation, top_view,
seed, steps, guidance_scale, controlnet_scale, weight_exponent, texture_size,
progress=gr.Progress(track_tqdm=True)):
import nvdiffrast.torch as dr
dev = "cuda"
glctx = dr.RasterizeCudaContext()
v = torch.from_numpy(verts).to(dev)
nrm = F.normalize(torch.from_numpy(normals).to(dev), dim=-1)
tri = torch.from_numpy(faces).to(dev)
uv = torch.from_numpy(uvs).to(dev)
T = int(texture_size)
# --- texel buffers (rasterize in UV space) ---
uv_clip = torch.cat([uv * 2 - 1, torch.zeros_like(uv[:, :1]), torch.ones_like(uv[:, :1])], -1)[None]
rast_uv, _ = dr.rasterize(glctx, uv_clip.contiguous(), tri, resolution=[T, T])
tex_mask = (rast_uv[..., 3] > 0)[0] # (T,T), row 0 = v 0
tex_pos = dr.interpolate(v[None], rast_uv, tri)[0][0]
tex_nrm = F.normalize(dr.interpolate(nrm[None], rast_uv, tri)[0][0], dim=-1)
proj = torch.from_numpy(perspective(FOV, 0.05, 100.0)).to(dev)
v_h = torch.cat([v, torch.ones_like(v[:, :1])], -1)
p_h = torch.cat([tex_pos, torch.ones_like(tex_pos[..., :1])], -1)
cams = []
for d in camera_directions(int(num_cameras), float(elevation), bool(top_view)):
dist = fit_camera(verts, d)
eye = d * dist
cams.append((eye, torch.from_numpy(look_at(eye)).to(dev)))
generator = torch.Generator(dev).manual_seed(int(seed))
colors, quals = [], [] # per camera: projected colour (T,T,3), visibility*|cos| (T,T)
gallery = []
exp = float(weight_exponent)
def blend():
if not colors:
return None, None
w = torch.stack(quals) ** exp
col = (w[..., None] * torch.stack(colors)).sum(0) / w.sum(0)[..., None].clamp(min=1e-8)
covered = (w.sum(0) > 0).float()
return col, covered
for ci, (eye, view) in enumerate(progress.tqdm(cams, desc="Generating views")):
mvp = proj @ view
clip = (v_h @ mvp.T)[None].contiguous()
rast, _ = dr.rasterize(glctx, clip, tri, resolution=[RES, RES])
rast = rast.flip(1) # top row first
fg = (rast[..., 3] > 0)[0]
vz = -(v_h @ view.T)[:, 2:3]
zbuf = dr.interpolate(vz[None].contiguous(), rast, tri)[0][0, ..., 0]
zbuf = torch.where(fg, zbuf, torch.full_like(zbuf, 1e6))
# depth guide: normalize over the object, invert (near = white), background black
zf = zbuf[fg]
depth = torch.zeros((RES, RES), device=dev)
depth[fg] = 1.0 - (zf - zf.min()) / (zf.max() - zf.min()).clamp(min=1e-6)
context = regen = None
if colors:
col, covered = blend()
ctx_tex = push_pull(col.permute(2, 0, 1)[None], covered[None, None])[0].permute(1, 2, 0)
cov = 1 - torch.prod(1 - torch.stack(quals), 0) # angle-aware coverage by earlier views
pix_uv = dr.interpolate(uv[None], rast, tri)[0][0]
grid = (pix_uv * 2 - 1)[None]
ctx = F.grid_sample(ctx_tex.permute(2, 0, 1)[None], grid, mode="bilinear", align_corners=False)[0]
cov_px = F.grid_sample(cov[None, None], grid, mode="nearest", align_corners=False)[0, 0]
context = torch.where(fg[..., None], ctx.permute(1, 2, 0), torch.full((RES, RES, 3), 0.5, device=dev))
# coverage ramp (low 0.15, high 1.0) -> keep; everything else is regenerated
keep = ((cov_px - 0.15) / (1.0 - 0.15)).clamp(0, 1) * fg.float()
regen = 1.0 - keep
regen = F.max_pool2d(regen[None, None], 7, stride=1, padding=3) # grow mask by 3 px
regen = gaussian_blur(regen, sigma=1.0, radius=1)[0, 0]
image = sample_view(prompt, negative_prompt, depth, context, regen, int(steps), float(guidance_scale),
float(controlnet_scale), generator)
gallery.append((to_pil(image), f"View {ci + 1}"))
gallery.append((to_pil(depth[..., None].expand(-1, -1, 3)), f"View {ci + 1} - depth guide"))
# --- project this view onto the texels ---
pc = p_h @ mvp.T
w_clip = pc[..., 3]
ndc = pc[..., :3] / w_clip[..., None].clamp(min=1e-6)
inside = (ndc[..., 0].abs() <= 1) & (ndc[..., 1].abs() <= 1) & (w_clip > 0) & tex_mask
grid = torch.stack([ndc[..., 0], -ndc[..., 1]], -1)[None]
z_map = F.grid_sample(zbuf[None, None], grid, mode="nearest", align_corners=False)[0, 0]
z_tex = -(p_h @ view.T)[..., 2]
visible = inside & (z_tex <= z_map + 0.02)
to_cam = F.normalize(torch.from_numpy(eye.astype(np.float32)).to(dev) - tex_pos, dim=-1)
cosang = (tex_nrm * to_cam).sum(-1).abs() # backface projection allowed, discard angle 90 deg
# soft silhouette guard: fade contributions from the outermost ~2 px of the generated object,
# so background colour at slightly misaligned edges is not projected onto the surface
fg_soft = -F.max_pool2d(-fg.float()[None, None], 5, stride=1, padding=2)
fg_soft = gaussian_blur(fg_soft, sigma=1.0, radius=2)
edge = F.grid_sample(fg_soft, grid, mode="bilinear", align_corners=False)[0, 0]
quals.append(torch.where(visible, cosang * edge, torch.zeros_like(cosang)))
col = F.grid_sample(image.permute(2, 0, 1)[None], grid, mode="bilinear", align_corners=False)[0]
colors.append(col.permute(1, 2, 0))
col, covered = blend()
final = push_pull(col.permute(2, 0, 1)[None], covered[None, None])[0].permute(1, 2, 0)
tex_img = to_pil(final.flip(0)) # image rows top-first (v = 1 at the top)
return tex_img, gallery
# ---------------------------------------------------------------------------------------------
# Gradio entry point
# ---------------------------------------------------------------------------------------------
def texture_mesh(mesh_file: str, prompt: str, negative_prompt: str = "", num_cameras: int = 4,
elevation: float = 20.0, top_view: bool = True, seed: int = 0, randomize_seed: bool = True,
steps: int = 8, guidance_scale: float = 1.5, controlnet_scale: float = 0.5,
weight_exponent: float = 3.0, texture_size: int = 1024,
progress=gr.Progress(track_tqdm=True)):
"""Texture a 3D mesh from a text prompt with StableGen's sequential SDXL projection workflow.
Args:
mesh_file: path to a .glb / .gltf / .obj / .ply / .stl mesh (existing materials are ignored).
prompt: description of the surface, e.g. "weathered bronze statue, green patina".
negative_prompt: optional negative prompt.
num_cameras: number of orbit cameras around the object (4-8).
elevation: elevation of the orbit cameras in degrees.
top_view: add an extra camera looking down from above.
seed: random seed.
randomize_seed: pick a random seed instead of `seed`.
steps: sampling steps per view (SDXL-Lightning 8-step LoRA).
guidance_scale: classifier-free guidance scale.
controlnet_scale: depth ControlNet strength.
weight_exponent: view-weight exponent for blending projections (higher = sharper seams).
texture_size: resolution of the baked UV texture.
Returns:
Textured GLB path, gallery of generated views and depth guides, and the used seed.
"""
if not mesh_file:
raise gr.Error("Upload a mesh first.")
if not prompt or not prompt.strip():
raise gr.Error("Enter a texture prompt.")
if randomize_seed:
seed = random.randint(0, 2**31 - 1)
t0 = time.perf_counter()
mesh = load_mesh(mesh_file)
center = (mesh.bounds[0] + mesh.bounds[1]) / 2
scale = float(np.linalg.norm(mesh.vertices - center, axis=1).max())
norm_mesh = trimesh.Trimesh((mesh.vertices - center) / scale, mesh.faces, process=False)
verts, normals, faces, uvs = unwrap(norm_mesh)
print(f"[prep] {len(faces)} faces, unwrap {time.perf_counter() - t0:.1f}s")
t1 = time.perf_counter()
tex_img, gallery = texture_mesh_gpu(verts, normals, faces, uvs, prompt, negative_prompt, int(num_cameras),
float(elevation), bool(top_view), int(seed), int(steps),
float(guidance_scale), float(controlnet_scale), float(weight_exponent),
int(texture_size))
print(f"[gpu] {time.perf_counter() - t1:.1f}s")
material = trimesh.visual.material.PBRMaterial(
baseColorTexture=tex_img, metallicFactor=0.0, roughnessFactor=1.0, doubleSided=True
)
out = trimesh.Trimesh(
vertices=verts * scale + center, faces=faces,
visual=trimesh.visual.TextureVisuals(uv=uvs, material=material), process=False,
)
path = tempfile.NamedTemporaryFile(suffix=".glb", delete=False).name
out.export(path)
return path, gallery, seed
CSS = """
#col-container { max-width: 1200px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""
with gr.Blocks() as demo:
with gr.Column(elem_id="col-container"):
gr.Markdown(
"""# StableGen: texture a 3D model from a prompt
A port of [StableGen](https://github.com/sakalond/StableGen)'s core texturing workflow (SDXL **Default** preset, *Generate Sequentially*).
Cameras are placed around the mesh; each view is generated with RealVisXL V5.0 + SDXL-Lightning, guided by a depth ControlNet,
and inpaints only what earlier views have not covered. The views are projected back onto the surface, blended by viewing angle,
and baked into a UV texture. [Wiki](https://github.com/sakalond/StableGen/wiki) · [Blender add-on](https://github.com/sakalond/StableGen/releases)"""
)
with gr.Row():
with gr.Column():
mesh_in = gr.Model3D(label="Input mesh (.glb / .obj / .ply / .stl)", height=380)
prompt = gr.Textbox(label="Texture prompt", placeholder="weathered bronze statue, green patina")
run = gr.Button("Generate texture", variant="primary")
with gr.Accordion("Advanced settings", open=False):
negative_prompt = gr.Textbox(label="Negative prompt", value="")
with gr.Row():
num_cameras = gr.Slider(4, 8, value=4, step=1, label="Number of orbit cameras")
elevation = gr.Slider(-30, 60, value=20, step=1, label="Camera elevation (deg)")
top_view = gr.Checkbox(value=True, label="Add top camera")
with gr.Row():
seed = gr.Number(value=0, precision=0, label="Seed")
randomize_seed = gr.Checkbox(value=True, label="Randomize seed")
with gr.Row():
steps = gr.Slider(4, 12, value=8, step=1, label="Steps")
guidance_scale = gr.Slider(1.0, 3.0, value=1.5, step=0.1, label="CFG")
with gr.Row():
controlnet_scale = gr.Slider(0.0, 1.0, value=0.5, step=0.05, label="Depth ControlNet strength")
weight_exponent = gr.Slider(1, 15, value=3, step=0.5, label="View weight exponent")
texture_size = gr.Radio([1024, 2048], value=1024, label="Texture resolution")
with gr.Column():
mesh_out = gr.Model3D(label="Textured mesh (GLB)", height=480)
views = gr.Gallery(label="Generated views and depth guides", columns=4, height=300)
inputs = [mesh_in, prompt, negative_prompt, num_cameras, elevation, top_view, seed, randomize_seed, steps,
guidance_scale, controlnet_scale, weight_exponent, texture_size]
run.click(texture_mesh, inputs=inputs, outputs=[mesh_out, views, seed], api_name="texture_mesh")
gr.Examples(
examples=[
["examples/suzanne.glb", "weathered bronze statue, green patina"],
["examples/lantern.glb", "old rusty iron lantern with brass details, worn paint"],
["examples/barramundi_fish.glb", "a realistic barramundi fish, silver scales, golden fins"],
["examples/corset.glb", "victorian corset, dark red velvet with gold embroidery"],
],
inputs=[mesh_in, prompt],
outputs=[mesh_out, views, seed],
fn=texture_mesh,
cache_examples=True,
cache_mode="lazy",
)
demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)