akhaliq HF Staff commited on
Commit
c66ee68
·
verified ·
1 Parent(s): d4efcb1

Gradio demo: Depth Pro 2 image and video metric depth

Browse files
Files changed (1) hide show
  1. app.py +139 -0
app.py ADDED
@@ -0,0 +1,139 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import tempfile
4
+ from functools import partial
5
+ from pathlib import Path
6
+
7
+ import av
8
+ import gradio as gr
9
+ import matplotlib
10
+ import matplotlib.colormaps
11
+ import numpy as np
12
+ import torch
13
+ from PIL import Image
14
+
15
+ try:
16
+ import spaces # ZeroGPU: available when the Space runs on ZeroGPU hardware
17
+ except ImportError:
18
+ spaces = None
19
+
20
+ from depth_pro_2 import DepthPro2, load_rgb, upright
21
+ from depth_pro_2.model import to_normalized_tensor
22
+
23
+ MODEL_ID = "apple/DepthPro2-H"
24
+ MAX_VIDEO_FRAMES = 100
25
+
26
+ # The weights load at startup; they are moved to CUDA inside the GPU-decorated
27
+ # functions, since ZeroGPU exposes the GPU only during those calls.
28
+ model = DepthPro2.from_pretrained(
29
+ MODEL_ID, precision=torch.bfloat16 if torch.cuda.is_available() else torch.float32
30
+ )
31
+
32
+
33
+ def _gpu(duration: int):
34
+ """ZeroGPU wrapper; a no-op where the spaces package is absent (plain CPU)."""
35
+ if spaces is not None and hasattr(spaces, "GPU"):
36
+ return spaces.GPU(duration=duration)
37
+ return lambda fn: fn
38
+
39
+
40
+ def _device_and_model():
41
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
42
+ return model.to(device), partial(to_normalized_tensor, device=device)
43
+
44
+
45
+ def _colorize(depth: np.ndarray) -> np.ndarray:
46
+ """Metric depth in meters to RGB, red near and blue far; white where infinitely far."""
47
+ finite = np.isfinite(depth)
48
+ lo, hi = np.percentile(depth[finite], [2, 98])
49
+ norm = np.clip((depth - lo) / max(hi - lo, 1e-6), 0.0, 1.0)
50
+ rgb = matplotlib.colormaps["Spectral"](norm)[..., :3]
51
+ rgb[~finite] = 1.0
52
+ return (rgb * 255).astype(np.uint8)
53
+
54
+
55
+ @_gpu(120)
56
+ def run_image(image: Image.Image) -> Image.Image:
57
+ with tempfile.TemporaryDirectory() as tmp:
58
+ path = Path(tmp) / "input.png"
59
+ image.save(path)
60
+ img, f_px = load_rgb(path) # f_px: the focal length recorded in the photo, or None
61
+ net, transform = _device_and_model()
62
+ depth = net.infer(transform(img), f_px=f_px)["depth"] # meters, (H, W)
63
+ return Image.fromarray(_colorize(depth.float().cpu().numpy()))
64
+
65
+
66
+ @_gpu(300)
67
+ def run_video(video: str) -> str:
68
+ container = av.open(video)
69
+ rate = container.streams.video[0].average_rate
70
+ fps = int(round(float(rate))) if rate else 24
71
+
72
+ def frames():
73
+ for frame in container.decode(video=0):
74
+ yield upright(frame)
75
+
76
+ net, transform = _device_and_model()
77
+ predictions = net.infer_video(transform(f) for f in frames()) # one per frame, in order
78
+
79
+ out_path = Path(tempfile.gettempdir()) / "depthpro2-output.mp4"
80
+ writer = None
81
+ written = 0
82
+ with av.open(out_path, mode="w") as out:
83
+ for prediction in predictions:
84
+ depth = prediction["depth"].float().cpu().numpy()
85
+ rgb = _colorize(depth)
86
+ if writer is None:
87
+ height, width = rgb.shape[:2]
88
+ writer = out.add_stream("libx264", rate=fps)
89
+ writer.width, writer.height = width - width % 2, height - height % 2
90
+ writer.pix_fmt = "yuv420p"
91
+ frame = av.VideoFrame.from_ndarray(rgb[: writer.height, : writer.width], format="rgb24")
92
+ for packet in writer.encode(frame):
93
+ out.mux(packet)
94
+ written += 1
95
+ if written >= MAX_VIDEO_FRAMES:
96
+ break
97
+ if writer is not None:
98
+ for packet in writer.encode():
99
+ out.mux(packet)
100
+ if writer is None:
101
+ raise gr.Error("Could not decode any frames from that video.")
102
+ return str(out_path)
103
+
104
+
105
+ TITLE = "# Depth Pro 2 — metric monocular depth"
106
+ DESCRIPTION = (
107
+ "[apple/DepthPro2](https://huggingface.co/apple/DepthPro2) estimates metric depth for every "
108
+ "pixel from a single photo or a whole video. Depth is sharp at object edges and steady from "
109
+ "frame to frame; red is near, blue is far, and white marks the infinitely far (sky). "
110
+ "For research use only."
111
+ )
112
+
113
+ with gr.Blocks(title="Depth Pro 2") as demo:
114
+ gr.Markdown(TITLE + "\n\n" + DESCRIPTION)
115
+ with gr.Tabs():
116
+ with gr.Tab("Image"):
117
+ with gr.Row():
118
+ image_in = gr.Image(type="pil", label="Input photo")
119
+ image_out = gr.Image(label="Predicted depth", interactive=False)
120
+ gr.Button("Estimate depth", variant="primary").click(
121
+ run_image, inputs=image_in, outputs=image_out
122
+ )
123
+ gr.Examples(
124
+ [
125
+ [
126
+ "https://huggingface.co/apple/DepthPro2/resolve/main/assets/teaser.jpg"
127
+ ]
128
+ ],
129
+ inputs=image_in,
130
+ )
131
+ with gr.Tab("Video"):
132
+ with gr.Row():
133
+ video_in = gr.Video(label="Input video (first 100 frames)")
134
+ video_out = gr.Video(label="Predicted depth", interactive=False)
135
+ gr.Button("Estimate depth", variant="primary").click(
136
+ run_video, inputs=video_in, outputs=video_out
137
+ )
138
+
139
+ demo.launch()