File size: 10,094 Bytes
c66ee68 6df2866 e6ce011 c66ee68 ffd1a97 c66ee68 d1846d3 b9f01b8 c66ee68 54be9ba c66ee68 54be9ba c66ee68 afa74dd 54be9ba afa74dd d1846d3 babd6b4 d1846d3 afa74dd c66ee68 b9f01b8 1feea84 b9f01b8 afa74dd b9f01b8 c66ee68 1feea84 c66ee68 afa74dd b9f01b8 afa74dd c66ee68 84501a5 b9f01b8 c66ee68 afa74dd 59ee84a b9f01b8 c66ee68 59ee84a c66ee68 59ee84a c66ee68 e1950cc 59ee84a c66ee68 59ee84a b9f01b8 c66ee68 59ee84a c66ee68 59ee84a c66ee68 b9f01b8 59ee84a c66ee68 b9f01b8 c66ee68 b9f01b8 6df2866 ffd1a97 ca006c6 6df2866 b9f01b8 e6ce011 85b3388 e6ce011 b9f01b8 85b3388 59ee84a b9f01b8 d1846d3 b9f01b8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 | from __future__ import annotations
import base64
import io
from functools import partial
from pathlib import Path
import av
import gradio as gr
import numpy as np
import torch
from fastapi.responses import FileResponse, HTMLResponse
from gradio import Server
from gradio.data_classes import FileData
from PIL import Image
try:
import spaces # ZeroGPU: available when the Space runs on ZeroGPU hardware
except ImportError:
spaces = None
from depth_pro_2 import DepthPro2, load_rgb, upright
from depth_pro_2.model import to_normalized_tensor
from depth_pro_2.network.fov import FOVEstimator
from huggingface_hub import hf_hub_download
from safetensors.torch import load_file
MAX_VIDEO_FRAMES = 60
MAX_IMAGE_SIDE = 1536
MAX_VIDEO_SIDE = 960
# Weights load from the Hub and go to CUDA at module level: ZeroGPU's CUDA emulation allows
# placements at startup, and lazy .to("cuda") inside @spaces.GPU is discouraged (docs:
# huggingface.co/docs/hub/spaces-zerogpu, Model loading) — transfers are optimized for startup.
# The backbone comes from apple/DepthPro2-H (whose config has "fov": false, so the focal
# estimator is attached separately from apple/DepthPro2-focal, its own repo). Hub downloads
# resume instead of corrupting, unlike the preset loader's CDN checkpoints. bfloat16 is how the
# models run on a GPU.
model = DepthPro2.from_pretrained("apple/DepthPro2-H", precision=torch.bfloat16)
model.fov = FOVEstimator(precision=torch.bfloat16)
model.fov.load_state_dict(
load_file(hf_hub_download("apple/DepthPro2-focal", "model.safetensors"))
)
model.eval().to("cuda")
# Self-host the @gradio/client bundle: browsers on some networks cannot reach public CDNs, and
# the page's dynamic import is what failed for them. Download it once at startup (server-side
# package installs work here) and serve it from this Space's own origin; the page falls back to
# the CDNs only if this file is missing.
_CLIENT_SOURCES = [
"https://cdn.jsdelivr.net/npm/@gradio/client@2.7.1/dist/index.min.js",
"https://unpkg.com/@gradio/client@2.7.1/dist/index.min.js",
"https://cdn.statically.io/npm/@gradio/client@2.7.1/dist/index.min.js",
]
_client_path = Path(__file__).resolve().parent / "gradio_client.min.js"
if not _client_path.exists():
import urllib.request
for _src in _CLIENT_SOURCES:
try:
urllib.request.urlretrieve(_src, _client_path)
break
except OSError:
continue
def _resize(img: np.ndarray, f_px: float | None, max_side: int):
"""Downscale so the longer side is at most ``max_side``, scaling ``f_px`` to match."""
height, width = img.shape[:2]
scale = max_side / max(height, width)
if scale >= 1.0:
return img, f_px
out = Image.fromarray(img).resize((round(width * scale), round(height * scale)), Image.LANCZOS)
return np.asarray(out), None if f_px is None else f_px * scale
# ColorBrewer "Spectral" stops, dark red (near) to violet (far), interpolated to a 256-entry LUT.
_SPECTRAL = np.array(
[
[0x9E, 0x01, 0x42], [0xD5, 0x3E, 0x4F], [0xF4, 0x6D, 0x43], [0xFD, 0xAE, 0x61],
[0xFE, 0xE0, 0x8B], [0xFF, 0xFF, 0xBF], [0xE6, 0xF5, 0x98], [0xAB, 0xDD, 0xA4],
[0x66, 0xC2, 0xA5], [0x32, 0x88, 0xBD], [0x5E, 0x4F, 0xA2],
],
dtype=np.float64,
) / 255.0
_LUT = np.stack(
[
np.interp(np.linspace(0.0, 1.0, 256), np.linspace(0.0, 1.0, len(_SPECTRAL)), _SPECTRAL[:, c])
for c in range(3)
],
axis=1,
)
def _gpu(duration: int):
"""ZeroGPU wrapper; a no-op where the spaces package is absent (plain CPU)."""
if spaces is not None and hasattr(spaces, "GPU"):
return spaces.GPU(duration=duration)
return lambda fn: fn
def _device_and_model():
return model, partial(to_normalized_tensor, device=torch.device("cuda"))
def _colorize(depth: np.ndarray) -> np.ndarray:
"""Metric depth in meters to RGB, red near and blue far; white where infinitely far."""
finite = np.isfinite(depth)
lo, hi = np.percentile(depth[finite], [2, 98])
norm = np.clip((depth - lo) / max(hi - lo, 1e-6), 0.0, 1.0)
rgb = _LUT[(norm * 255).astype(np.uint8)]
rgb[~finite] = 1.0
return (rgb * 255).astype(np.uint8)
@_gpu(60)
def _infer_image(img: np.ndarray, f_px: float | None) -> np.ndarray:
"""Depth colorization for one photo, downscaled to at most 1536 px on the longer side."""
img, f_px = _resize(img, f_px, MAX_IMAGE_SIDE)
net, transform = _device_and_model()
prediction = net.infer(transform(img), f_px=f_px)
if prediction["depth"] is None:
raise ValueError("No focal length recorded in the photo and none could be estimated.")
return _colorize(prediction["depth"].float().cpu().numpy())
@_gpu(240)
def _infer_video(path: str) -> dict:
"""Depth colorization for a video (first 60 frames, up to 960 px); returns source + depth data URIs.
The source is re-encoded to H.264 from the already-decoded frames: phone videos are often
HEVC, which Chrome cannot decode, and an unplayable input window otherwise reads as if the
depth result failed. The frames are already in memory, so the second encode is cheap.
"""
container = av.open(path)
rate = container.streams.video[0].average_rate
fps = int(round(float(rate))) if rate else 24
imgs = []
for frame in container.decode(video=0):
imgs.append(_resize(upright(frame), None, MAX_VIDEO_SIDE)[0])
if len(imgs) >= MAX_VIDEO_FRAMES:
break
if not imgs:
raise ValueError("Could not decode any frames from that video.")
net, transform = _device_and_model()
predictions = net.infer_video(transform(f) for f in imgs)
out_path = Path(path).with_suffix(".depth.mp4")
src_path = Path(path).with_suffix(".source.mp4")
written = 0
with av.open(out_path, mode="w") as out, av.open(src_path, mode="w") as sout:
writer = None
src_writer = None
for img, prediction in zip(imgs, predictions):
rgb = _colorize(prediction["depth"].float().cpu().numpy())
if writer is None:
height, width = rgb.shape[:2]
writer = out.add_stream("libx264", rate=fps)
writer.width, writer.height = width - width % 2, height - height % 2
writer.pix_fmt = "yuv420p"
# The source at the depth map's size, so both windows encode the same frames.
src_writer = sout.add_stream("libx264", rate=fps)
src_writer.width, src_writer.height = writer.width, writer.height
src_writer.pix_fmt = "yuv420p"
for stream, muxer, pixels in ((writer, out, rgb), (src_writer, sout, img)):
frame = av.VideoFrame.from_ndarray(
np.ascontiguousarray(pixels[: writer.height, : writer.width]), format="rgb24"
)
for packet in stream.encode(frame):
muxer.mux(packet)
written += 1
if writer is not None:
for packet in writer.encode():
out.mux(packet)
for packet in src_writer.encode():
sout.mux(packet)
if writer is None:
raise ValueError("Could not decode any frames from that video.")
print(f"infer_video: {written} frames, depth {out_path.stat().st_size} B, source {src_path.stat().st_size} B", flush=True)
return {
"image_data": "data:video/mp4;base64," + base64.b64encode(out_path.read_bytes()).decode(),
"source": "data:video/mp4;base64," + base64.b64encode(src_path.read_bytes()).decode(),
}
app = Server(
title="Depth Pro 2",
description="Metric monocular depth from photos and videos (apple/DepthPro2).",
)
@app.api(name="predict_depth")
def predict_depth(image: FileData, request: gr.Request) -> str:
"""Estimate metric depth for an uploaded photo; returns it inline as a data URI."""
# request is taken explicitly: Server-mode ZeroGPU token forwarding is more reliable with it
# in the signature (see MiniMaxAI/MiniMax-H3-Turbo-Lora commit 7db57ff).
try:
img, f_px = load_rgb(image["path"]) # f_px: the focal length recorded in the photo, or None
except Exception as err:
raise gr.Error(
f"That upload could not be read as an image ({type(err).__name__}) \u2014 try a JPG or PNG."
) from err
print(f"predict_depth: {img.shape[1]}x{img.shape[0]} px, f_px={f_px}", flush=True)
rgb = _infer_image(img, f_px)
# Encode at display resolution (long side 1024): the depth map is for viewing, and a compact
# data URI sidesteps any size limits between the API and the browser.
out = Image.fromarray(rgb)
if max(out.size) > 1024:
scale = 1024 / max(out.size)
out = out.resize((round(out.width * scale), round(out.height * scale)), Image.LANCZOS)
buffer = io.BytesIO()
out.save(buffer, "PNG")
print(f"predict_depth: encoded {len(buffer.getvalue())} bytes", flush=True)
# A dict (not a bare string) is the proven-safe return: gradio's postprocess leaves dicts
# without file metadata untouched, while a bare data-URI string can get mangled as a path
# (the P-Image-OSS pattern).
return {"image_data": "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()}
@app.api(name="predict_depth_video")
def predict_depth_video(video: FileData, request: gr.Request) -> dict:
"""Estimate metric depth for an uploaded video (first 60 frames); returns source + depth data URIs."""
return _infer_video(video["path"])
@app.get("/gradio-client.js")
def gradio_client_js():
"""The @gradio/client bundle, served same-origin so no CDN is needed in the browser."""
if _client_path.exists():
return FileResponse(_client_path, media_type="text/javascript")
return HTMLResponse("// client bundle not available", status_code=404)
@app.get("/", response_class=HTMLResponse)
async def homepage():
return (Path(__file__).resolve().parent / "index.html").read_text(encoding="utf-8")
app.launch(show_error=True)
|