from __future__ import annotations import base64 import io from functools import partial from pathlib import Path import av import gradio as gr import numpy as np import torch from fastapi.responses import FileResponse, HTMLResponse from gradio import Server from gradio.data_classes import FileData from PIL import Image try: import spaces # ZeroGPU: available when the Space runs on ZeroGPU hardware except ImportError: spaces = None from depth_pro_2 import DepthPro2, load_rgb, upright from depth_pro_2.model import to_normalized_tensor from depth_pro_2.network.fov import FOVEstimator from huggingface_hub import hf_hub_download from safetensors.torch import load_file MAX_VIDEO_FRAMES = 60 MAX_IMAGE_SIDE = 1536 MAX_VIDEO_SIDE = 960 # Weights load from the Hub and go to CUDA at module level: ZeroGPU's CUDA emulation allows # placements at startup, and lazy .to("cuda") inside @spaces.GPU is discouraged (docs: # huggingface.co/docs/hub/spaces-zerogpu, Model loading) — transfers are optimized for startup. # The backbone comes from apple/DepthPro2-H (whose config has "fov": false, so the focal # estimator is attached separately from apple/DepthPro2-focal, its own repo). Hub downloads # resume instead of corrupting, unlike the preset loader's CDN checkpoints. bfloat16 is how the # models run on a GPU. model = DepthPro2.from_pretrained("apple/DepthPro2-H", precision=torch.bfloat16) model.fov = FOVEstimator(precision=torch.bfloat16) model.fov.load_state_dict( load_file(hf_hub_download("apple/DepthPro2-focal", "model.safetensors")) ) model.eval().to("cuda") # Self-host the @gradio/client bundle: browsers on some networks cannot reach public CDNs, and # the page's dynamic import is what failed for them. Download it once at startup (server-side # package installs work here) and serve it from this Space's own origin; the page falls back to # the CDNs only if this file is missing. _CLIENT_SOURCES = [ "https://cdn.jsdelivr.net/npm/@gradio/client@2.7.1/dist/index.min.js", "https://unpkg.com/@gradio/client@2.7.1/dist/index.min.js", "https://cdn.statically.io/npm/@gradio/client@2.7.1/dist/index.min.js", ] _client_path = Path(__file__).resolve().parent / "gradio_client.min.js" if not _client_path.exists(): import urllib.request for _src in _CLIENT_SOURCES: try: urllib.request.urlretrieve(_src, _client_path) break except OSError: continue def _resize(img: np.ndarray, f_px: float | None, max_side: int): """Downscale so the longer side is at most ``max_side``, scaling ``f_px`` to match.""" height, width = img.shape[:2] scale = max_side / max(height, width) if scale >= 1.0: return img, f_px out = Image.fromarray(img).resize((round(width * scale), round(height * scale)), Image.LANCZOS) return np.asarray(out), None if f_px is None else f_px * scale # ColorBrewer "Spectral" stops, dark red (near) to violet (far), interpolated to a 256-entry LUT. _SPECTRAL = np.array( [ [0x9E, 0x01, 0x42], [0xD5, 0x3E, 0x4F], [0xF4, 0x6D, 0x43], [0xFD, 0xAE, 0x61], [0xFE, 0xE0, 0x8B], [0xFF, 0xFF, 0xBF], [0xE6, 0xF5, 0x98], [0xAB, 0xDD, 0xA4], [0x66, 0xC2, 0xA5], [0x32, 0x88, 0xBD], [0x5E, 0x4F, 0xA2], ], dtype=np.float64, ) / 255.0 _LUT = np.stack( [ np.interp(np.linspace(0.0, 1.0, 256), np.linspace(0.0, 1.0, len(_SPECTRAL)), _SPECTRAL[:, c]) for c in range(3) ], axis=1, ) def _gpu(duration: int): """ZeroGPU wrapper; a no-op where the spaces package is absent (plain CPU).""" if spaces is not None and hasattr(spaces, "GPU"): return spaces.GPU(duration=duration) return lambda fn: fn def _device_and_model(): return model, partial(to_normalized_tensor, device=torch.device("cuda")) def _colorize(depth: np.ndarray) -> np.ndarray: """Metric depth in meters to RGB, red near and blue far; white where infinitely far.""" finite = np.isfinite(depth) lo, hi = np.percentile(depth[finite], [2, 98]) norm = np.clip((depth - lo) / max(hi - lo, 1e-6), 0.0, 1.0) rgb = _LUT[(norm * 255).astype(np.uint8)] rgb[~finite] = 1.0 return (rgb * 255).astype(np.uint8) @_gpu(60) def _infer_image(img: np.ndarray, f_px: float | None) -> np.ndarray: """Depth colorization for one photo, downscaled to at most 1536 px on the longer side.""" img, f_px = _resize(img, f_px, MAX_IMAGE_SIDE) net, transform = _device_and_model() prediction = net.infer(transform(img), f_px=f_px) if prediction["depth"] is None: raise ValueError("No focal length recorded in the photo and none could be estimated.") return _colorize(prediction["depth"].float().cpu().numpy()) @_gpu(240) def _infer_video(path: str) -> dict: """Depth colorization for a video (first 60 frames, up to 960 px); returns source + depth data URIs. The source is re-encoded to H.264 from the already-decoded frames: phone videos are often HEVC, which Chrome cannot decode, and an unplayable input window otherwise reads as if the depth result failed. The frames are already in memory, so the second encode is cheap. """ container = av.open(path) rate = container.streams.video[0].average_rate fps = int(round(float(rate))) if rate else 24 imgs = [] for frame in container.decode(video=0): imgs.append(_resize(upright(frame), None, MAX_VIDEO_SIDE)[0]) if len(imgs) >= MAX_VIDEO_FRAMES: break if not imgs: raise ValueError("Could not decode any frames from that video.") net, transform = _device_and_model() predictions = net.infer_video(transform(f) for f in imgs) out_path = Path(path).with_suffix(".depth.mp4") src_path = Path(path).with_suffix(".source.mp4") written = 0 with av.open(out_path, mode="w") as out, av.open(src_path, mode="w") as sout: writer = None src_writer = None for img, prediction in zip(imgs, predictions): rgb = _colorize(prediction["depth"].float().cpu().numpy()) if writer is None: height, width = rgb.shape[:2] writer = out.add_stream("libx264", rate=fps) writer.width, writer.height = width - width % 2, height - height % 2 writer.pix_fmt = "yuv420p" # The source at the depth map's size, so both windows encode the same frames. src_writer = sout.add_stream("libx264", rate=fps) src_writer.width, src_writer.height = writer.width, writer.height src_writer.pix_fmt = "yuv420p" for stream, muxer, pixels in ((writer, out, rgb), (src_writer, sout, img)): frame = av.VideoFrame.from_ndarray( np.ascontiguousarray(pixels[: writer.height, : writer.width]), format="rgb24" ) for packet in stream.encode(frame): muxer.mux(packet) written += 1 if writer is not None: for packet in writer.encode(): out.mux(packet) for packet in src_writer.encode(): sout.mux(packet) if writer is None: raise ValueError("Could not decode any frames from that video.") print(f"infer_video: {written} frames, depth {out_path.stat().st_size} B, source {src_path.stat().st_size} B", flush=True) return { "image_data": "data:video/mp4;base64," + base64.b64encode(out_path.read_bytes()).decode(), "source": "data:video/mp4;base64," + base64.b64encode(src_path.read_bytes()).decode(), } app = Server( title="Depth Pro 2", description="Metric monocular depth from photos and videos (apple/DepthPro2).", ) @app.api(name="predict_depth") def predict_depth(image: FileData, request: gr.Request) -> str: """Estimate metric depth for an uploaded photo; returns it inline as a data URI.""" # request is taken explicitly: Server-mode ZeroGPU token forwarding is more reliable with it # in the signature (see MiniMaxAI/MiniMax-H3-Turbo-Lora commit 7db57ff). try: img, f_px = load_rgb(image["path"]) # f_px: the focal length recorded in the photo, or None except Exception as err: raise gr.Error( f"That upload could not be read as an image ({type(err).__name__}) \u2014 try a JPG or PNG." ) from err print(f"predict_depth: {img.shape[1]}x{img.shape[0]} px, f_px={f_px}", flush=True) rgb = _infer_image(img, f_px) # Encode at display resolution (long side 1024): the depth map is for viewing, and a compact # data URI sidesteps any size limits between the API and the browser. out = Image.fromarray(rgb) if max(out.size) > 1024: scale = 1024 / max(out.size) out = out.resize((round(out.width * scale), round(out.height * scale)), Image.LANCZOS) buffer = io.BytesIO() out.save(buffer, "PNG") print(f"predict_depth: encoded {len(buffer.getvalue())} bytes", flush=True) # A dict (not a bare string) is the proven-safe return: gradio's postprocess leaves dicts # without file metadata untouched, while a bare data-URI string can get mangled as a path # (the P-Image-OSS pattern). return {"image_data": "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()} @app.api(name="predict_depth_video") def predict_depth_video(video: FileData, request: gr.Request) -> dict: """Estimate metric depth for an uploaded video (first 60 frames); returns source + depth data URIs.""" return _infer_video(video["path"]) @app.get("/gradio-client.js") def gradio_client_js(): """The @gradio/client bundle, served same-origin so no CDN is needed in the browser.""" if _client_path.exists(): return FileResponse(_client_path, media_type="text/javascript") return HTMLResponse("// client bundle not available", status_code=404) @app.get("/", response_class=HTMLResponse) async def homepage(): return (Path(__file__).resolve().parent / "index.html").read_text(encoding="utf-8") app.launch(show_error=True)