import argparse import gradio as gr import json import os import tempfile from typing import Any import cv2 import numpy as np import torch from PIL import Image, ImageOps from sklearn.decomposition import PCA from stl import mesh from transformers import AutoModelForDepthEstimation, AutoProcessor try: import pillow_heif pillow_heif.register_heif_opener() HEIF_SUPPORT = True except Exception as heif_error: # pragma: no cover - surfaced in UI if HEIC open fails pillow_heif = None HEIF_SUPPORT = False HEIF_IMPORT_ERROR = heif_error else: HEIF_IMPORT_ERROR = None MIN_RESOLUTION_LIMIT = 10_000 DEFAULT_MAX_RESOLUTION = int(os.getenv("DEFAULT_MAX_RESOLUTION", "1500000")) MAX_RESOLUTION_LIMIT = int(os.getenv("MAX_RESOLUTION_LIMIT", "20000000")) def parse_launch_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Image to 3D Relief Gradio app") parser.add_argument( "--max-resolution-pixels", type=int, default=None, help=( "Maximum number of pixels processed for depth estimation. " "Default comes from DEFAULT_MAX_RESOLUTION or 1,500,000." ), ) parser.add_argument( "--server-name", default=os.getenv("GRADIO_SERVER_NAME", "0.0.0.0"), help="Host/interface for Gradio to bind.", ) parser.add_argument( "--server-port", type=int, default=int(os.getenv("PORT", os.getenv("GRADIO_SERVER_PORT", "7860"))), help="Port for Gradio to bind.", ) args, _ = parser.parse_known_args() return args LAUNCH_ARGS = parse_launch_args() REQUESTED_MAX_RESOLUTION = LAUNCH_ARGS.max_resolution_pixels or DEFAULT_MAX_RESOLUTION CONFIGURED_MAX_RESOLUTION = int(max( MIN_RESOLUTION_LIMIT, min(MAX_RESOLUTION_LIMIT, REQUESTED_MAX_RESOLUTION), )) MODEL_ID = "depth-anything/Depth-Anything-V2-Large-hf" DEFAULT_TEXTURE_STRENGTH = 0.1 DEFAULT_TEXTURE_SMOOTHING = 5 DEFAULT_MIN_Z = 0.5 DEFAULT_MAX_Z = 5.0 DEFAULT_CURVE_POINTS = [ {"x": 0.0, "y": 0.0}, {"x": 1.0, "y": 1.0}, ] ACCEPTED_IMAGE_TYPES = [ ".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tif", ".tiff", ".heic", ".heif", ] device = "cuda" if torch.cuda.is_available() else "cpu" processor = AutoProcessor.from_pretrained(MODEL_ID, use_fast=True) model = AutoModelForDepthEstimation.from_pretrained(MODEL_ID).to(device) model.eval() print(f"Model loaded successfully on {device}.") APP_CSS = """ #curve-points-json, #depth-histogram-json, #auto-preview-depth-btn { display: none !important; } .depth-editor-card { border: 1px solid var(--border-color-primary); border-radius: 12px; padding: 14px; background: var(--background-fill-secondary); } .depth-editor-toolbar { display: flex; flex-wrap: wrap; gap: 8px; margin: 8px 0 10px; } .depth-editor-toolbar button { border: 1px solid var(--border-color-primary); border-radius: 8px; padding: 6px 10px; background: var(--button-secondary-background-fill); color: var(--body-text-color); cursor: pointer; } .depth-editor-toolbar button:hover { background: var(--button-secondary-background-fill-hover); } #depth-curve-canvas { width: 100%; max-width: 720px; height: 360px; display: block; touch-action: none; border: 1px solid var(--border-color-primary); border-radius: 8px; background: #111827; } .depth-editor-help { margin-top: 8px; color: var(--body-text-color-subdued); font-size: 0.9rem; line-height: 1.35; } .depth-editor-readout { min-height: 1.2rem; margin-top: 6px; color: var(--body-text-color-subdued); font-family: var(--font-mono); font-size: 0.85rem; } """ CURVE_EDITOR_HEAD = """ """ CURVE_EDITOR_HTML = """
Depth histogram + curves editor
Drag control points to remap the normalized height interpretation before STL export. The curve uses piecewise cubic Bézier segments. Click the curve to add a point. Click an already selected point to toggle smooth/corner continuity. Corner points draw as diamonds. Double-click a non-endpoint, or select it and press Delete, to remove it. The blue histogram shows the current smoothed base-depth distribution.
""" def clamp(value: float, minimum: float, maximum: float) -> float: return max(minimum, min(maximum, value)) def normalize_array(values: np.ndarray) -> np.ndarray: values = values.astype(np.float32, copy=False) v_min = float(np.nanmin(values)) v_max = float(np.nanmax(values)) if not np.isfinite(v_min) or not np.isfinite(v_max) or v_max <= v_min: return np.zeros_like(values, dtype=np.float32) return ((values - v_min) / (v_max - v_min)).astype(np.float32) def sanitize_max_pixels(max_pixels: Any) -> int: try: requested = int(float(max_pixels)) except (TypeError, ValueError): requested = DEFAULT_MAX_RESOLUTION return int(clamp(requested, MIN_RESOLUTION_LIMIT, MAX_RESOLUTION_LIMIT)) def coerce_float(value: Any, default: float) -> float: try: if value is None: return float(default) return float(value) except (TypeError, ValueError): return float(default) def coerce_z_bounds(min_z: Any, max_z: Any) -> tuple[float, float]: min_z_value = coerce_float(min_z, DEFAULT_MIN_Z) max_z_value = coerce_float(max_z, DEFAULT_MAX_Z) return min_z_value, max_z_value def get_lanczos_resample_filter() -> int: return getattr(getattr(Image, "Resampling", Image), "LANCZOS") def load_and_resize_image(input_filepath: str, max_pixels: int) -> tuple[np.ndarray, str]: if input_filepath is None: raise gr.Error("Please upload an image.") filepath = str(input_filepath) filepath_lower = filepath.lower() try: image_pil = Image.open(filepath) image_pil = ImageOps.exif_transpose(image_pil).convert("RGB") except Exception as exc: heif_hint = "" if filepath_lower.endswith((".heic", ".heif")): if HEIF_SUPPORT: heif_hint = " The file looks like HEIC/HEIF; pillow-heif is installed but could not decode this file." else: heif_hint = f" HEIC/HEIF support is unavailable because pillow-heif failed to load: {HEIF_IMPORT_ERROR}" raise gr.Error(f"Could not open image file.{heif_hint} Error: {exc}") from exc original_w, original_h = image_pil.size original_pixels = original_w * original_h max_pixels = sanitize_max_pixels(max_pixels) resized = False if original_pixels > max_pixels: ratio = (max_pixels / original_pixels) ** 0.5 new_w = max(1, int(original_w * ratio)) new_h = max(1, int(original_h * ratio)) image_pil = image_pil.resize((new_w, new_h), get_lanczos_resample_filter()) resized = True input_image = np.array(image_pil, dtype=np.uint8) current_h, current_w = input_image.shape[:2] status = ( f"Loaded {original_w}×{original_h} image. " f"Processing at {current_w}×{current_h} ({current_w * current_h:,} pixels)." ) if resized: status += f" Downsampled to stay under the {max_pixels:,}-pixel limit." if filepath_lower.endswith((".heic", ".heif")): status += " HEIC/HEIF decoded server-side." return input_image, status def estimate_depth_map(input_image: np.ndarray) -> np.ndarray: image = Image.fromarray(input_image).convert("RGB") with torch.no_grad(): inputs = processor(images=image, return_tensors="pt").to(device) outputs = model(**inputs) predicted_depth = outputs.predicted_depth depth = torch.nn.functional.interpolate( predicted_depth.unsqueeze(1), size=(image.height, image.width), mode="bilinear", align_corners=False, ).squeeze().cpu().numpy() return normalize_depth_for_emboss(depth) def apply_smoothing(values: np.ndarray, smoothing: int) -> np.ndarray: try: ksize = int(smoothing) except (TypeError, ValueError): ksize = 0 if ksize <= 1: return values.astype(np.float32, copy=True) if ksize % 2 == 0: ksize += 1 return cv2.GaussianBlur(values.astype(np.float32), (ksize, ksize), 0).astype(np.float32) def sanitize_curve_points(curve_points_json: str | None) -> list[dict[str, Any]]: try: raw_points = json.loads(curve_points_json or "") except (TypeError, json.JSONDecodeError): raw_points = DEFAULT_CURVE_POINTS points: list[dict[str, Any]] = [] if isinstance(raw_points, list): for point in raw_points: mode = "smooth" if isinstance(point, dict): x, y = point.get("x"), point.get("y") raw_mode = str(point.get("mode", "smooth")).lower() if raw_mode in {"corner", "broken", "linear"}: mode = "corner" elif isinstance(point, (list, tuple)) and len(point) >= 2: x, y = point[0], point[1] else: continue try: x_f = clamp(float(x), 0.0, 1.0) y_f = clamp(float(y), 0.0, 1.0) except (TypeError, ValueError): continue points.append({"x": x_f, "y": y_f, "mode": mode}) if len(points) < 2: points = [{**point, "mode": "smooth"} for point in DEFAULT_CURVE_POINTS] points.sort(key=lambda point: float(point["x"])) deduped: list[dict[str, Any]] = [] for point in points: if deduped and abs(float(point["x"]) - float(deduped[-1]["x"])) < 1e-5: deduped[-1] = point else: deduped.append(point) if len(deduped) < 2: deduped = [{**point, "mode": "smooth"} for point in DEFAULT_CURVE_POINTS] deduped[0]["x"] = 0.0 deduped[-1]["x"] = 1.0 deduped[0]["mode"] = "smooth" deduped[-1]["mode"] = "smooth" return deduped def curve_points_to_json(points: list[dict[str, Any]]) -> str: return json.dumps([ { "x": round(float(point["x"]), 5), "y": round(float(point["y"]), 5), "mode": "corner" if str(point.get("mode", "smooth")) == "corner" else "smooth", } for point in points ]) def pchip_endpoint_slope(h0: float, h1: float, delta0: float, delta1: float) -> float: slope = ((2.0 * h0 + h1) * delta0 - h0 * delta1) / max(h0 + h1, 1e-6) if slope == 0.0 or delta0 == 0.0 or np.sign(slope) != np.sign(delta0): return 0.0 if np.sign(delta0) != np.sign(delta1) and abs(slope) > abs(3.0 * delta0): return float(3.0 * delta0) return float(slope) def cubic_bezier_curve_data(points: list[dict[str, Any]]) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: x_values = np.array([point["x"] for point in points], dtype=np.float32) y_values = np.array([point["y"] for point in points], dtype=np.float32) interval_count = len(points) - 1 if interval_count <= 0: return x_values, y_values, np.zeros_like(x_values), np.zeros_like(x_values) h_values = np.maximum(np.diff(x_values), 1e-6) secant_slopes = np.diff(y_values) / h_values shared_slopes = np.zeros(len(points), dtype=np.float32) if len(points) == 2: shared_slopes[:] = secant_slopes[0] else: shared_slopes[0] = pchip_endpoint_slope(float(h_values[0]), float(h_values[1]), float(secant_slopes[0]), float(secant_slopes[1])) shared_slopes[-1] = pchip_endpoint_slope(float(h_values[-1]), float(h_values[-2]), float(secant_slopes[-1]), float(secant_slopes[-2])) for i in range(1, len(points) - 1): left_delta = float(secant_slopes[i - 1]) right_delta = float(secant_slopes[i]) if left_delta == 0.0 or right_delta == 0.0 or np.sign(left_delta) != np.sign(right_delta): shared_slopes[i] = 0.0 else: left_h = float(h_values[i - 1]) right_h = float(h_values[i]) w1 = 2.0 * right_h + left_h w2 = right_h + 2.0 * left_h shared_slopes[i] = float((w1 + w2) / ((w1 / left_delta) + (w2 / right_delta))) left_slopes = shared_slopes.copy() right_slopes = shared_slopes.copy() for i in range(1, len(points) - 1): if str(points[i].get("mode", "smooth")) == "corner": left_slopes[i] = secant_slopes[i - 1] right_slopes[i] = secant_slopes[i] return x_values, y_values, left_slopes.astype(np.float32), right_slopes.astype(np.float32) def evaluate_piecewise_cubic_bezier(values: np.ndarray, points: list[dict[str, Any]]) -> np.ndarray: x_values, y_values, left_slopes, right_slopes = cubic_bezier_curve_data(points) if len(points) <= 1: return np.full_like(values, y_values[0] if len(y_values) else 0.0, dtype=np.float32) flat_values = values.astype(np.float32, copy=False).reshape(-1) interval_indices = np.searchsorted(x_values, flat_values, side="right") - 1 interval_indices = np.clip(interval_indices, 0, len(x_values) - 2) interval_widths = np.maximum(x_values[interval_indices + 1] - x_values[interval_indices], 1e-6) t_values = np.clip((flat_values - x_values[interval_indices]) / interval_widths, 0.0, 1.0) omt_values = 1.0 - t_values p0 = y_values[interval_indices] p1 = y_values[interval_indices] + right_slopes[interval_indices] * (interval_widths / 3.0) p2 = y_values[interval_indices + 1] - left_slopes[interval_indices + 1] * (interval_widths / 3.0) p3 = y_values[interval_indices + 1] adjusted = ( (omt_values ** 3) * p0 + 3.0 * (omt_values ** 2) * t_values * p1 + 3.0 * omt_values * (t_values ** 2) * p2 + (t_values ** 3) * p3 ) return adjusted.reshape(values.shape).astype(np.float32) def apply_depth_curve(depth_map: np.ndarray, curve_points_json: str | None, invert_depth: bool) -> np.ndarray: working = depth_map.astype(np.float32, copy=True) if invert_depth: working = 1.0 - working points = sanitize_curve_points(curve_points_json) adjusted = evaluate_piecewise_cubic_bezier(working, points) return np.clip(adjusted, 0.0, 1.0) def depth_to_preview_image(depth_map: np.ndarray) -> Image.Image: preview = np.clip(depth_map, 0.0, 1.0) preview_uint8 = np.rint(np.clip(preview, 0.0, 1.0) * 255.0).astype(np.uint8) return Image.fromarray(preview_uint8) def histogram_json(depth_map: np.ndarray, bins: int = 96) -> str: counts, _ = np.histogram(np.clip(depth_map, 0.0, 1.0), bins=bins, range=(0.0, 1.0)) return json.dumps(counts.astype(int).tolist()) def prepare_depth(input_filepath: str): input_image, load_status = load_and_resize_image(input_filepath, CONFIGURED_MAX_RESOLUTION) depth_normalized = estimate_depth_map(input_image) curve_json = curve_points_to_json(DEFAULT_CURVE_POINTS) adjusted_depth = apply_depth_curve(depth_normalized, curve_json, invert_depth=False) height_map = combine_depth_and_texture( input_image, adjusted_depth, DEFAULT_TEXTURE_STRENGTH, DEFAULT_TEXTURE_SMOOTHING, ) state = { "image_rgb": input_image, "raw_depth": depth_normalized, "adjusted_depth": adjusted_depth, "height_map": height_map, "curve_points_json": curve_json, "depth_map_smoothing": 0, "invert_depth": False, "texture_strength": DEFAULT_TEXTURE_STRENGTH, "texture_smoothing": DEFAULT_TEXTURE_SMOOTHING, "load_status": load_status, } status = ( f"{load_status}\nDepth estimation complete. Use the curve and luminance texture controls to tune the 2D height map before generating the STL." ) return ( state, Image.fromarray(input_image), depth_to_preview_image(depth_normalized), depth_to_preview_image(height_map), histogram_json(depth_normalized), curve_json, status, "Preview is using the linear curve with the default luminance texture settings.", ) def clear_depth_outputs(): return ( None, None, None, None, "[]", curve_points_to_json(DEFAULT_CURVE_POINTS), "Upload an image to estimate its depth map.", "", ) def clear_preview_outputs(message: str = "Upload an image to estimate its depth map before tuning preview controls."): return ( None, None, "[]", curve_points_to_json(DEFAULT_CURVE_POINTS), message, ) def auto_prepare_depth(input_filepath: str): if input_filepath is None: return clear_depth_outputs() return prepare_depth(input_filepath) def get_smoothed_depth_from_state(depth_state: dict[str, Any] | None, depth_map_smoothing: int) -> np.ndarray: if not depth_state or "raw_depth" not in depth_state: raise gr.Error("Please upload an image and estimate its depth first.") return normalize_array(apply_smoothing(depth_state["raw_depth"], depth_map_smoothing)) def update_depth_preview( depth_state: dict[str, Any] | None, curve_points_json: str | None, depth_map_smoothing: int, invert_depth: bool, texture_strength: float, texture_smoothing: int, ): if not depth_state or "raw_depth" not in depth_state: return clear_preview_outputs() texture_strength = coerce_float(texture_strength, DEFAULT_TEXTURE_STRENGTH) texture_smoothing = int(coerce_float(texture_smoothing, DEFAULT_TEXTURE_SMOOTHING)) smoothed_depth = get_smoothed_depth_from_state(depth_state, depth_map_smoothing) adjusted_depth = apply_depth_curve(smoothed_depth, curve_points_json, invert_depth) points = sanitize_curve_points(curve_points_json) normalized_curve_json = curve_points_to_json(points) height_map = combine_depth_and_texture( depth_state["image_rgb"], adjusted_depth, texture_strength, texture_smoothing, ) if depth_state is not None: depth_state["adjusted_depth"] = adjusted_depth depth_state["height_map"] = height_map depth_state["curve_points_json"] = normalized_curve_json depth_state["depth_map_smoothing"] = int(depth_map_smoothing) depth_state["invert_depth"] = bool(invert_depth) depth_state["texture_strength"] = float(texture_strength) depth_state["texture_smoothing"] = int(texture_smoothing) status = ( f"Preview updated with {len(points)} curve points, texture strength {float(texture_strength):.3f}, " f"texture smoothing {int(texture_smoothing)}." ) return ( depth_state, depth_to_preview_image(height_map), histogram_json(smoothed_depth), normalized_curve_json, status, ) def get_adjusted_depth_for_settings( depth_state: dict[str, Any], curve_points_json: str | None, depth_map_smoothing: int, invert_depth: bool, ) -> np.ndarray: normalized_curve_json = curve_points_to_json(sanitize_curve_points(curve_points_json)) cached_depth = depth_state.get("adjusted_depth") cache_matches = ( cached_depth is not None and depth_state.get("curve_points_json") == normalized_curve_json and int(depth_state.get("depth_map_smoothing", -1)) == int(depth_map_smoothing) and bool(depth_state.get("invert_depth", False)) == bool(invert_depth) ) if cache_matches: return cached_depth smoothed_depth = get_smoothed_depth_from_state(depth_state, depth_map_smoothing) adjusted_depth = apply_depth_curve(smoothed_depth, normalized_curve_json, invert_depth) depth_state["adjusted_depth"] = adjusted_depth depth_state["curve_points_json"] = normalized_curve_json depth_state["depth_map_smoothing"] = int(depth_map_smoothing) depth_state["invert_depth"] = bool(invert_depth) return adjusted_depth def get_height_map_for_settings( depth_state: dict[str, Any], curve_points_json: str | None, depth_map_smoothing: int, invert_depth: bool, texture_strength: float, texture_smoothing: int, ) -> np.ndarray: texture_strength = coerce_float(texture_strength, DEFAULT_TEXTURE_STRENGTH) texture_smoothing = int(coerce_float(texture_smoothing, DEFAULT_TEXTURE_SMOOTHING)) normalized_curve_json = curve_points_to_json(sanitize_curve_points(curve_points_json)) cached_height_map = depth_state.get("height_map") cache_matches = ( cached_height_map is not None and depth_state.get("curve_points_json") == normalized_curve_json and int(depth_state.get("depth_map_smoothing", -1)) == int(depth_map_smoothing) and bool(depth_state.get("invert_depth", False)) == bool(invert_depth) and float(depth_state.get("texture_strength", -1.0)) == float(texture_strength) and int(depth_state.get("texture_smoothing", -1)) == int(texture_smoothing) ) if cache_matches: return cached_height_map adjusted_depth = get_adjusted_depth_for_settings( depth_state, normalized_curve_json, depth_map_smoothing, invert_depth, ) height_map = combine_depth_and_texture( depth_state["image_rgb"], adjusted_depth, texture_strength, texture_smoothing, ) depth_state["height_map"] = height_map depth_state["texture_strength"] = float(texture_strength) depth_state["texture_smoothing"] = int(texture_smoothing) return height_map def combine_depth_and_texture( image_rgb: np.ndarray, adjusted_depth: np.ndarray, texture_strength: float, texture_smoothing: int, ) -> np.ndarray: texture_strength = clamp(coerce_float(texture_strength, DEFAULT_TEXTURE_STRENGTH), 0.0, 0.5) if texture_strength <= 0.0: return np.clip(adjusted_depth, 0.0, 1.0).astype(np.float32) gray_image = cv2.cvtColor(image_rgb, cv2.COLOR_RGB2GRAY).astype(np.float32) / 255.0 low_frequency = apply_smoothing(gray_image, 5) detail = gray_image - low_frequency texture_smoothing = int(coerce_float(texture_smoothing, DEFAULT_TEXTURE_SMOOTHING)) if texture_smoothing > 1: detail = apply_smoothing(detail, texture_smoothing) if detail.shape != adjusted_depth.shape: detail = cv2.resize( detail, (adjusted_depth.shape[1], adjusted_depth.shape[0]), interpolation=cv2.INTER_LINEAR, ) scale = float(np.percentile(np.abs(detail), 95.0)) if not np.isfinite(scale) or scale <= 1e-6: detail = np.zeros_like(adjusted_depth, dtype=np.float32) else: detail = np.clip(detail / scale, -1.0, 1.0).astype(np.float32) combined_map = adjusted_depth + (detail * texture_strength) return np.clip(combined_map, 0.0, 1.0).astype(np.float32) def apply_pca_correction_to_z(z_data: np.ndarray, x_length: float, min_z: float, max_z: float) -> np.ndarray: height, width = z_data.shape y_length = x_length * (height / width) x_coords_1d = np.linspace(0, x_length, width) y_coords_1d = np.linspace(y_length, 0, height) x_grid, y_grid = np.meshgrid(x_coords_1d, y_coords_1d) points = np.stack([x_grid.flatten(), y_grid.flatten(), z_data.flatten()], axis=1) n_points = points.shape[0] n_samples = min(n_points, 50_000) sample_indices = np.random.choice(n_points, n_samples, replace=False) pca = PCA(n_components=3) pca.fit(points[sample_indices]) normal = pca.components_[2] if normal[2] < 0: normal *= -1 p0 = pca.mean_ z_plane = p0[2] - (normal[0] * (x_grid - p0[0]) + normal[1] * (y_grid - p0[1])) / normal[2] corrected_z = z_data - z_plane corrected_normalized = normalize_array(corrected_z) return min_z + corrected_normalized * (max_z - min_z) def build_stl_faces(z_data: np.ndarray, x_length: float, close_body: bool) -> tuple[np.ndarray, float]: height, width = z_data.shape y_length = float(x_length * (height / width)) z_data = z_data.astype(np.float32, copy=False) x_coords = np.linspace(0, x_length, width, dtype=np.float32) y_coords = np.linspace(y_length, 0, height, dtype=np.float32) x_grid, y_grid = np.meshgrid(x_coords, y_coords) vertices = np.stack([x_grid, y_grid, z_data], axis=-1) tl = vertices[:-1, :-1] bl = vertices[1:, :-1] br = vertices[1:, 1:] tr = vertices[:-1, 1:] quad_count = (height - 1) * (width - 1) top_faces = np.empty((quad_count * 2, 3, 3), dtype=np.float32) top_faces[0::2] = np.stack([tl, bl, br], axis=2).reshape(-1, 3, 3) top_faces[1::2] = np.stack([tl, br, tr], axis=2).reshape(-1, 3, 3) if not close_body: return top_faces, y_length v_tl = vertices[0, 0] v_tr = vertices[0, width - 1] v_bl = vertices[height - 1, 0] v_br = vertices[height - 1, width - 1] b_tl = np.array([v_tl[0], v_tl[1], 0], dtype=np.float32) b_tr = np.array([v_tr[0], v_tr[1], 0], dtype=np.float32) b_bl = np.array([v_bl[0], v_bl[1], 0], dtype=np.float32) b_br = np.array([v_br[0], v_br[1], 0], dtype=np.float32) side_faces = np.array([ [v_tl, b_tl, b_tr], [v_tl, b_tr, v_tr], [v_br, b_br, b_bl], [v_br, b_bl, v_bl], [v_bl, b_bl, b_tl], [v_bl, b_tl, v_tl], [v_tr, b_tr, b_br], [v_tr, b_br, v_br], ], dtype=np.float32) base_faces = np.array([ [b_tl, b_br, b_bl], [b_tl, b_tr, b_br], ], dtype=np.float32) return np.concatenate([top_faces, side_faces, base_faces], axis=0), y_length def flip_faces_for_preview(faces: np.ndarray, y_length: float) -> np.ndarray: preview_faces = faces.copy() preview_faces[..., 1] = np.float32(y_length) - preview_faces[..., 1] preview_faces = preview_faces[:, [0, 2, 1], :] return preview_faces def save_stl_faces(faces: np.ndarray) -> str: surface = mesh.Mesh(np.zeros(faces.shape[0], dtype=mesh.Mesh.dtype)) surface.vectors = faces with tempfile.NamedTemporaryFile(delete=False, suffix=".stl") as tmpfile: surface.save(tmpfile.name) return tmpfile.name def build_stl_mesh(z_data: np.ndarray, x_length: float, close_body: bool, flip_for_preview: bool = False) -> str: faces, y_length = build_stl_faces(z_data, x_length, close_body) if flip_for_preview: faces = flip_faces_for_preview(faces, y_length) return save_stl_faces(faces) def generate_3d_model_from_adjusted_depth( depth_state: dict[str, Any] | None, curve_points_json: str | None, depth_map_smoothing: int, invert_depth: bool, texture_strength: float, texture_smoothing: int, max_z: float, min_z: float, x_length: float, do_pca_correction: bool, close_body: bool, ): min_z, max_z = coerce_z_bounds(min_z, max_z) x_length = coerce_float(x_length, 100.0) if max_z <= min_z: raise gr.Error("Max Z-height must be greater than Min Z-height.") if x_length <= 0: raise gr.Error("X Length must be positive.") if not depth_state or "image_rgb" not in depth_state: raise gr.Error("Please estimate a depth map before generating an STL.") height_map = get_height_map_for_settings( depth_state, curve_points_json, depth_map_smoothing, invert_depth, texture_strength, texture_smoothing, ) z_data = min_z + height_map * (max_z - min_z) if do_pca_correction: z_data = apply_pca_correction_to_z(z_data, x_length, min_z, max_z) export_faces, y_length = build_stl_faces(z_data, x_length, close_body) export_stl_path = save_stl_faces(export_faces) preview_stl_path = save_stl_faces(flip_faces_for_preview(export_faces, y_length)) point_count = len(sanitize_curve_points(curve_points_json)) status = ( f"Generated STL from previewed height map using {point_count} curve points and texture strength " f"{float(texture_strength):.3f}." ) return preview_stl_path, export_stl_path, status # CONTROLLED_EMBOSS_V1 DEFAULT_HEIGHT_GAMMA = 1.0 DEFAULT_BACKGROUND_FLATTEN = 0.08 def normalize_depth_for_emboss(values: np.ndarray) -> np.ndarray: values = values.astype(np.float32, copy=False) finite_values = values[np.isfinite(values)] if finite_values.size == 0: return np.zeros_like(values, dtype=np.float32) low, high = np.nanpercentile(finite_values, [2.0, 98.0]) if not np.isfinite(low) or not np.isfinite(high) or high <= low: return normalize_array(values) return np.clip((values - low) / (high - low), 0.0, 1.0).astype(np.float32) def apply_emboss_controls(height_map: np.ndarray, height_gamma: float, background_flatten: float) -> np.ndarray: gamma = clamp(coerce_float(height_gamma, DEFAULT_HEIGHT_GAMMA), 0.25, 4.0) flatten = clamp(coerce_float(background_flatten, DEFAULT_BACKGROUND_FLATTEN), 0.0, 0.8) controlled = np.power(np.clip(height_map, 0.0, 1.0), gamma).astype(np.float32) if flatten > 0.0: controlled = np.clip((controlled - flatten) / max(1.0 - flatten, 1e-6), 0.0, 1.0) return controlled.astype(np.float32) def height_to_hillshade_image(height_map: np.ndarray) -> Image.Image: heights = np.clip(height_map, 0.0, 1.0).astype(np.float32) gradient_y, gradient_x = np.gradient(heights) normals = np.dstack((-gradient_x * 4.0, -gradient_y * 4.0, np.ones_like(heights))) normals /= np.maximum(np.linalg.norm(normals, axis=2, keepdims=True), 1e-6) light = np.array([-0.45, -0.35, 0.82], dtype=np.float32) light /= np.linalg.norm(light) shade = 0.18 + 0.82 * np.tensordot(normals, light, axes=([2], [0])) return Image.fromarray(np.rint(np.clip(shade, 0.0, 1.0) * 255.0).astype(np.uint8)) def save_height_map_png(height_map: np.ndarray) -> str: height_uint16 = np.rint(np.clip(height_map, 0.0, 1.0) * 65535.0).astype(np.uint16) with tempfile.NamedTemporaryFile(delete=False, suffix=".png") as tmpfile: Image.fromarray(height_uint16, mode="I;16").save(tmpfile.name, format="PNG") return tmpfile.name def build_controlled_height_map( image_rgb: np.ndarray, adjusted_depth: np.ndarray, texture_strength: float, texture_smoothing: int, height_gamma: float, background_flatten: float, ) -> np.ndarray: controlled_depth = apply_emboss_controls(adjusted_depth, height_gamma, background_flatten) return combine_depth_and_texture(image_rgb, controlled_depth, texture_strength, texture_smoothing) def estimate_depth_map(input_image: np.ndarray) -> np.ndarray: image = Image.fromarray(input_image).convert("RGB") with torch.no_grad(): inputs = processor(images=image, return_tensors="pt").to(device) outputs = model(**inputs) predicted_depth = outputs.predicted_depth depth = torch.nn.functional.interpolate( predicted_depth.unsqueeze(1), size=(image.height, image.width), mode="bilinear", align_corners=False, ).squeeze().cpu().numpy() return normalize_depth_for_emboss(depth) def prepare_depth(input_filepath: str): input_image, load_status = load_and_resize_image(input_filepath, CONFIGURED_MAX_RESOLUTION) depth_normalized = estimate_depth_map(input_image) curve_json = curve_points_to_json(DEFAULT_CURVE_POINTS) adjusted_depth = apply_depth_curve(depth_normalized, curve_json, invert_depth=False) height_map = build_controlled_height_map( input_image, adjusted_depth, DEFAULT_TEXTURE_STRENGTH, DEFAULT_TEXTURE_SMOOTHING, DEFAULT_HEIGHT_GAMMA, DEFAULT_BACKGROUND_FLATTEN, ) state = { "image_rgb": input_image, "raw_depth": depth_normalized, "adjusted_depth": adjusted_depth, "height_map": height_map, "curve_points_json": curve_json, "depth_map_smoothing": 0, "invert_depth": False, "texture_strength": DEFAULT_TEXTURE_STRENGTH, "texture_smoothing": DEFAULT_TEXTURE_SMOOTHING, "height_gamma": DEFAULT_HEIGHT_GAMMA, "background_flatten": DEFAULT_BACKGROUND_FLATTEN, "load_status": load_status, } status = ( f"{load_status}\nDepth estimation complete. Tune gamma and background flatten on the true grayscale height map before adding fine detail." ) return ( state, Image.fromarray(input_image), depth_to_preview_image(depth_normalized), depth_to_preview_image(height_map), height_to_hillshade_image(height_map), save_height_map_png(height_map), histogram_json(depth_normalized), curve_json, status, "Preview uses robust depth normalization, linear gamma, and the default background flatten threshold.", ) def clear_depth_outputs(): return ( None, None, None, None, None, None, "[]", curve_points_to_json(DEFAULT_CURVE_POINTS), "Upload an image to estimate its depth map.", "", ) def clear_preview_outputs(message: str = "Upload an image to estimate its depth map before tuning preview controls."): return ( None, None, None, None, "[]", curve_points_to_json(DEFAULT_CURVE_POINTS), message, ) def get_smoothed_depth_from_state(depth_state: dict[str, Any] | None, depth_map_smoothing: int) -> np.ndarray: if not depth_state or "raw_depth" not in depth_state: raise gr.Error("Please upload an image and estimate its depth first.") return normalize_depth_for_emboss(apply_smoothing(depth_state["raw_depth"], depth_map_smoothing)) def update_depth_preview( depth_state: dict[str, Any] | None, curve_points_json: str | None, depth_map_smoothing: int, invert_depth: bool, texture_strength: float, texture_smoothing: int, height_gamma: float, background_flatten: float, ): if not depth_state or "raw_depth" not in depth_state: return clear_preview_outputs() texture_strength = clamp(coerce_float(texture_strength, DEFAULT_TEXTURE_STRENGTH), 0.0, 0.5) texture_smoothing = int(coerce_float(texture_smoothing, DEFAULT_TEXTURE_SMOOTHING)) height_gamma = clamp(coerce_float(height_gamma, DEFAULT_HEIGHT_GAMMA), 0.25, 4.0) background_flatten = clamp(coerce_float(background_flatten, DEFAULT_BACKGROUND_FLATTEN), 0.0, 0.8) smoothed_depth = get_smoothed_depth_from_state(depth_state, depth_map_smoothing) adjusted_depth = apply_depth_curve(smoothed_depth, curve_points_json, invert_depth) points = sanitize_curve_points(curve_points_json) normalized_curve_json = curve_points_to_json(points) height_map = build_controlled_height_map( depth_state["image_rgb"], adjusted_depth, texture_strength, texture_smoothing, height_gamma, background_flatten, ) depth_state["adjusted_depth"] = adjusted_depth depth_state["height_map"] = height_map depth_state["curve_points_json"] = normalized_curve_json depth_state["depth_map_smoothing"] = int(depth_map_smoothing) depth_state["invert_depth"] = bool(invert_depth) depth_state["texture_strength"] = float(texture_strength) depth_state["texture_smoothing"] = int(texture_smoothing) depth_state["height_gamma"] = float(height_gamma) depth_state["background_flatten"] = float(background_flatten) status = ( f"Controlled preview updated with gamma {height_gamma:.2f}, background flatten {background_flatten:.2f}, " f"and fine detail strength {texture_strength:.3f}." ) return ( depth_state, depth_to_preview_image(height_map), height_to_hillshade_image(height_map), save_height_map_png(height_map), histogram_json(smoothed_depth), normalized_curve_json, status, ) def get_adjusted_depth_for_settings( depth_state: dict[str, Any], curve_points_json: str | None, depth_map_smoothing: int, invert_depth: bool, ) -> np.ndarray: normalized_curve_json = curve_points_to_json(sanitize_curve_points(curve_points_json)) cached_depth = depth_state.get("adjusted_depth") cache_matches = ( cached_depth is not None and depth_state.get("curve_points_json") == normalized_curve_json and int(depth_state.get("depth_map_smoothing", -1)) == int(depth_map_smoothing) and bool(depth_state.get("invert_depth", False)) == bool(invert_depth) ) if cache_matches: return cached_depth smoothed_depth = get_smoothed_depth_from_state(depth_state, depth_map_smoothing) adjusted_depth = apply_depth_curve(smoothed_depth, normalized_curve_json, invert_depth) depth_state["adjusted_depth"] = adjusted_depth depth_state["curve_points_json"] = normalized_curve_json depth_state["depth_map_smoothing"] = int(depth_map_smoothing) depth_state["invert_depth"] = bool(invert_depth) return adjusted_depth def get_height_map_for_settings( depth_state: dict[str, Any], curve_points_json: str | None, depth_map_smoothing: int, invert_depth: bool, texture_strength: float, texture_smoothing: int, height_gamma: float, background_flatten: float, ) -> np.ndarray: texture_strength = clamp(coerce_float(texture_strength, DEFAULT_TEXTURE_STRENGTH), 0.0, 0.5) texture_smoothing = int(coerce_float(texture_smoothing, DEFAULT_TEXTURE_SMOOTHING)) height_gamma = clamp(coerce_float(height_gamma, DEFAULT_HEIGHT_GAMMA), 0.25, 4.0) background_flatten = clamp(coerce_float(background_flatten, DEFAULT_BACKGROUND_FLATTEN), 0.0, 0.8) normalized_curve_json = curve_points_to_json(sanitize_curve_points(curve_points_json)) cached_height_map = depth_state.get("height_map") cache_matches = ( cached_height_map is not None and depth_state.get("curve_points_json") == normalized_curve_json and int(depth_state.get("depth_map_smoothing", -1)) == int(depth_map_smoothing) and bool(depth_state.get("invert_depth", False)) == bool(invert_depth) and float(depth_state.get("texture_strength", -1.0)) == float(texture_strength) and int(depth_state.get("texture_smoothing", -1)) == int(texture_smoothing) and float(depth_state.get("height_gamma", -1.0)) == float(height_gamma) and float(depth_state.get("background_flatten", -1.0)) == float(background_flatten) ) if cache_matches: return cached_height_map adjusted_depth = get_adjusted_depth_for_settings( depth_state, normalized_curve_json, depth_map_smoothing, invert_depth, ) height_map = build_controlled_height_map( depth_state["image_rgb"], adjusted_depth, texture_strength, texture_smoothing, height_gamma, background_flatten, ) depth_state["height_map"] = height_map depth_state["texture_strength"] = float(texture_strength) depth_state["texture_smoothing"] = int(texture_smoothing) depth_state["height_gamma"] = float(height_gamma) depth_state["background_flatten"] = float(background_flatten) return height_map def generate_3d_model_from_adjusted_depth( depth_state: dict[str, Any] | None, curve_points_json: str | None, depth_map_smoothing: int, invert_depth: bool, texture_strength: float, texture_smoothing: int, height_gamma: float, background_flatten: float, max_z: float, min_z: float, x_length: float, do_pca_correction: bool, close_body: bool, ): min_z, max_z = coerce_z_bounds(min_z, max_z) x_length = coerce_float(x_length, 100.0) if max_z <= min_z: raise gr.Error("Max Z-height must be greater than Min Z-height.") if x_length <= 0: raise gr.Error("X Length must be positive.") if not depth_state or "image_rgb" not in depth_state: raise gr.Error("Please estimate a depth map before generating an STL.") height_map = get_height_map_for_settings( depth_state, curve_points_json, depth_map_smoothing, invert_depth, texture_strength, texture_smoothing, height_gamma, background_flatten, ) z_data = min_z + height_map * (max_z - min_z) if do_pca_correction: z_data = apply_pca_correction_to_z(z_data, x_length, min_z, max_z) export_faces, y_length = build_stl_faces(z_data, x_length, close_body) export_stl_path = save_stl_faces(export_faces) preview_stl_path = save_stl_faces(flip_faces_for_preview(export_faces, y_length)) point_count = len(sanitize_curve_points(curve_points_json)) status = ( f"Generated STL from controlled height map using {point_count} curve points, gamma {float(height_gamma):.2f}, " f"and background flatten {float(background_flatten):.2f}." ) return preview_stl_path, export_stl_path, status # PUBLIC_TEST_EMBOSS_V1 TEST_LINEAR_CURVE = [ {"x": 0.0, "y": 0.0}, {"x": 1.0, "y": 1.0}, ] TEST_COMPRESS_HIGH_CURVE = [ {"x": 0.0, "y": 0.0}, {"x": 0.45, "y": 0.52}, {"x": 0.75, "y": 0.74}, {"x": 1.0, "y": 0.86}, ] def prepare_public_test_outputs( input_filepath: str | None, version: str, texture_strength: float, ) -> tuple[Image.Image | None, Image.Image | None, str]: if not input_filepath: return None, None, "Upload an image first." input_image, _ = load_and_resize_image(input_filepath, CONFIGURED_MAX_RESOLUTION) depth_map = estimate_depth_map(input_image) if version == "Version 2 — Compress high values": curve_points = TEST_COMPRESS_HIGH_CURVE else: curve_points = TEST_LINEAR_CURVE adjusted_depth = apply_depth_curve( depth_map, curve_points_to_json(curve_points), invert_depth=False, ) height_map = build_controlled_height_map( input_image, adjusted_depth, texture_strength, DEFAULT_TEXTURE_SMOOTHING, DEFAULT_HEIGHT_GAMMA, DEFAULT_BACKGROUND_FLATTEN, ) return ( depth_to_preview_image(height_map), height_to_hillshade_image(height_map), f"Completed {version} with Fine Detail Strength {float(texture_strength):.3f}.", ) with gr.Blocks(title="Test Embossed Effect") as demo: gr.Markdown("# Test Embossed Effect") gr.Markdown("Upload one image and compare the true grayscale height map with its embossed surface preview.") input_image = gr.Image( type="filepath", label="Upload one image", ) version = gr.Radio( choices=["Version 1 — Linear", "Version 2 — Compress high values"], value="Version 1 — Linear", label="Height interpretation version", ) texture_strength = gr.Slider( minimum=0.0, maximum=0.5, value=DEFAULT_TEXTURE_STRENGTH, step=0.001, label="Fine Detail Strength", info="Adds zero-mean fine source detail. Default uses strength 0.1 and smoothing 5.", ) rerun_button = gr.Button("Run / Rerun Preview", variant="primary") run_status = gr.Textbox( value="Upload an image, choose a version, then click Run / Rerun Preview.", label="Run status", interactive=False, ) with gr.Row(): height_map_output = gr.Image( format="png", image_mode="L", label="True grayscale height map", interactive=False, ) hillshade_output = gr.Image( format="png", image_mode="L", label="Embossed surface preview", interactive=False, ) test_inputs = [input_image, version, texture_strength] test_outputs = [height_map_output, hillshade_output, run_status] input_image.change( fn=prepare_public_test_outputs, inputs=test_inputs, outputs=test_outputs, show_progress="full", ) rerun_button.click( fn=prepare_public_test_outputs, inputs=test_inputs, outputs=test_outputs, show_progress="full", ) if __name__ == "__main__": demo.queue().launch( server_name=LAUNCH_ARGS.server_name, server_port=LAUNCH_ARGS.server_port, theme="base", )