ritianyu commited on
Commit
3c2ae9a
·
1 Parent(s): 9d7e197
InfiniDepth/utils/hf_demo_utils.py CHANGED
@@ -12,9 +12,8 @@ from huggingface_hub import hf_hub_download
12
 
13
  from .inference_utils import (
14
  build_scaled_intrinsics_matrix,
15
- has_missing_intrinsics,
16
  prepare_metric_depth_inputs,
17
- resolve_camera_intrinsics,
18
  resolve_output_size_from_mode,
19
  )
20
  from .io_utils import depth2pcd, depth_to_disparity
@@ -143,9 +142,29 @@ class DemoResult:
143
  rgb: np.ndarray
144
  depth_source_label: str
145
  output_size: tuple[int, int]
 
146
  used_default_intrinsics: bool
147
 
148
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
149
  class ModelCache:
150
  def __init__(self):
151
  self._cache: dict[tuple[str, str], Any] = {}
@@ -207,10 +226,18 @@ def _resolve_depth_inputs(
207
  input_size: tuple[int, int],
208
  image: torch.Tensor,
209
  device: torch.device,
210
- ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, str]:
 
 
 
 
 
 
 
 
211
  input_depth_path = depth_path if depth_path else None
212
  moge2_pretrained = resolve_moge2_pretrained()
213
- gt_depth, prompt_depth, gt_depth_mask, used_input_depth = prepare_metric_depth_inputs(
214
  input_depth_path=input_depth_path,
215
  input_size=input_size,
216
  image=image,
@@ -220,7 +247,7 @@ def _resolve_depth_inputs(
220
  )
221
  prompt_mask = prompt_depth > 0
222
  depth_source_label = "uploaded depth" if used_input_depth else "MoGe-2 prior"
223
- return gt_depth, prompt_depth, gt_depth_mask, prompt_mask, depth_source_label
224
 
225
 
226
  def _colorize_predicted_depth(pred_depthmap: torch.Tensor) -> np.ndarray:
@@ -357,7 +384,7 @@ def run_single_image_demo(
357
  if model_type == "InfiniDepth_DC":
358
  assert depth_path is not None and os.path.exists(depth_path), "InfiniDepth_DC requires a valid input depth map for depth completion. Please provide --input_depth_path."
359
 
360
- gt_depth, prompt_depth, gt_depth_mask, prompt_mask, depth_source_label = _resolve_depth_inputs(
361
  depth_path=depth_path,
362
  input_size=input_size,
363
  image=image,
@@ -392,7 +419,18 @@ def run_single_image_demo(
392
  depth_vis = _colorize_predicted_depth(pred_depthmap)
393
  _report_stage(stage_callback, "demo:depth_colorized")
394
 
395
- fx, fy, cx, cy = resolve_camera_intrinsics(fx_org, fy_org, cx_org, cy_org, org_h, org_w)
 
 
 
 
 
 
 
 
 
 
 
396
  fx_out, fy_out, cx_out, cy_out, _ = build_scaled_intrinsics_matrix(
397
  fx_org=fx,
398
  fy_org=fy,
@@ -434,5 +472,186 @@ def run_single_image_demo(
434
  rgb=rgb,
435
  depth_source_label=depth_source_label,
436
  output_size=(h_out, w_out),
437
- used_default_intrinsics=has_missing_intrinsics(fx_org, fy_org, cx_org, cy_org),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
438
  )
 
12
 
13
  from .inference_utils import (
14
  build_scaled_intrinsics_matrix,
 
15
  prepare_metric_depth_inputs,
16
+ resolve_camera_intrinsics_for_inference,
17
  resolve_output_size_from_mode,
18
  )
19
  from .io_utils import depth2pcd, depth_to_disparity
 
142
  rgb: np.ndarray
143
  depth_source_label: str
144
  output_size: tuple[int, int]
145
+ intrinsics_source_label: str
146
  used_default_intrinsics: bool
147
 
148
 
149
+ @dataclass
150
+ class GPUInferenceResult:
151
+ """Raw GPU inference outputs, all transferred to CPU numpy arrays."""
152
+ pred_depthmap_np: np.ndarray
153
+ query_coord_np: np.ndarray
154
+ pred_depth_np: np.ndarray
155
+ image_tensor_np: np.ndarray
156
+ depth_source_label: str
157
+ intrinsics_source_label: str
158
+ h_out: int
159
+ w_out: int
160
+ org_h: int
161
+ org_w: int
162
+ fx_out: float
163
+ fy_out: float
164
+ cx_out: float
165
+ cy_out: float
166
+
167
+
168
  class ModelCache:
169
  def __init__(self):
170
  self._cache: dict[tuple[str, str], Any] = {}
 
226
  input_size: tuple[int, int],
227
  image: torch.Tensor,
228
  device: torch.device,
229
+ ) -> tuple[
230
+ torch.Tensor,
231
+ torch.Tensor,
232
+ torch.Tensor,
233
+ torch.Tensor,
234
+ str,
235
+ str,
236
+ Optional[tuple[float, float, float, float]],
237
+ ]:
238
  input_depth_path = depth_path if depth_path else None
239
  moge2_pretrained = resolve_moge2_pretrained()
240
+ gt_depth, prompt_depth, gt_depth_mask, used_input_depth, moge2_intrinsics = prepare_metric_depth_inputs(
241
  input_depth_path=input_depth_path,
242
  input_size=input_size,
243
  image=image,
 
247
  )
248
  prompt_mask = prompt_depth > 0
249
  depth_source_label = "uploaded depth" if used_input_depth else "MoGe-2 prior"
250
+ return gt_depth, prompt_depth, gt_depth_mask, prompt_mask, depth_source_label, moge2_pretrained, moge2_intrinsics
251
 
252
 
253
  def _colorize_predicted_depth(pred_depthmap: torch.Tensor) -> np.ndarray:
 
384
  if model_type == "InfiniDepth_DC":
385
  assert depth_path is not None and os.path.exists(depth_path), "InfiniDepth_DC requires a valid input depth map for depth completion. Please provide --input_depth_path."
386
 
387
+ gt_depth, prompt_depth, gt_depth_mask, prompt_mask, depth_source_label, moge2_pretrained, moge2_intrinsics = _resolve_depth_inputs(
388
  depth_path=depth_path,
389
  input_size=input_size,
390
  image=image,
 
419
  depth_vis = _colorize_predicted_depth(pred_depthmap)
420
  _report_stage(stage_callback, "demo:depth_colorized")
421
 
422
+ fx, fy, cx, cy, intrinsics_source_label = resolve_camera_intrinsics_for_inference(
423
+ fx_org=fx_org,
424
+ fy_org=fy_org,
425
+ cx_org=cx_org,
426
+ cy_org=cy_org,
427
+ org_h=org_h,
428
+ org_w=org_w,
429
+ image=image,
430
+ moge2_pretrained=moge2_pretrained,
431
+ moge2_intrinsics=moge2_intrinsics,
432
+ )
433
+ Log.info(f"Camera intrinsics source resolved: {intrinsics_source_label}")
434
  fx_out, fy_out, cx_out, cy_out, _ = build_scaled_intrinsics_matrix(
435
  fx_org=fx,
436
  fy_org=fy,
 
472
  rgb=rgb,
473
  depth_source_label=depth_source_label,
474
  output_size=(h_out, w_out),
475
+ intrinsics_source_label=intrinsics_source_label,
476
+ used_default_intrinsics=intrinsics_source_label == "default",
477
+ )
478
+
479
+
480
+ def run_gpu_inference(
481
+ image_np: np.ndarray,
482
+ depth_path: Optional[str],
483
+ model_type: str,
484
+ input_size_text: str,
485
+ output_resolution_mode: str,
486
+ upsample_ratio: int,
487
+ fx_org: Optional[float] = None,
488
+ fy_org: Optional[float] = None,
489
+ cx_org: Optional[float] = None,
490
+ cy_org: Optional[float] = None,
491
+ model_cache: Optional[ModelCache] = None,
492
+ stage_callback: Optional[Callable[[str], None]] = None,
493
+ ) -> GPUInferenceResult:
494
+ """Run only GPU-intensive inference. All outputs are moved to CPU numpy before return.
495
+
496
+ This is designed for HuggingFace ZeroGPU where GPU time is limited: only the actual
497
+ CUDA work (MoGe-2, model inference, intrinsics estimation) runs here. CPU-heavy
498
+ post-processing (colorization, point cloud, file I/O) should happen outside the
499
+ ``@spaces.GPU`` decorated caller.
500
+ """
501
+ image_shape = tuple(int(d) for d in image_np.shape) if image_np is not None else None
502
+ _report_stage(stage_callback, "gpu:start")
503
+ Log.info(
504
+ f"run_gpu_inference: model_type={model_type}, input_size={input_size_text}, "
505
+ f"output_resolution_mode={output_resolution_mode}, upsample_ratio={upsample_ratio}, "
506
+ f"has_depth={bool(depth_path)}, image_shape={image_shape}, "
507
+ f"cuda_available={torch.cuda.is_available()}"
508
+ )
509
+ if not torch.cuda.is_available():
510
+ raise RuntimeError(
511
+ "No CUDA GPU is available. If using Hugging Face ZeroGPU, "
512
+ "decorate the Gradio inference function with @spaces.GPU and enable queue()."
513
+ )
514
+
515
+ input_size = _parse_image_size(input_size_text)
516
+ if upsample_ratio < 1 or upsample_ratio > 8:
517
+ raise ValueError("upsample_ratio must be in [1, 8]")
518
+ output_size = input_size
519
+ device = torch.device("cuda")
520
+
521
+ _debug = os.getenv("INFINIDEPTH_DEBUG_GPU", "0") == "1"
522
+
523
+ image, org_h, org_w = _prepare_image_tensor(image_np, input_size, device)
524
+ _report_stage(stage_callback, "gpu:image_prepared")
525
+ if _debug:
526
+ torch.cuda.synchronize()
527
+ Log.info(f"[GPU-DEBUG] image_prepared: GPU mem allocated={torch.cuda.memory_allocated(device) / 1e6:.1f}MB")
528
+
529
+ h_in, w_in = input_size
530
+ h_out, w_out = resolve_output_size_from_mode(
531
+ output_resolution_mode=output_resolution_mode,
532
+ org_h=org_h, org_w=org_w, h=h_in, w=w_in,
533
+ output_size=output_size, upsample_ratio=upsample_ratio,
534
+ )
535
+
536
+ if model_type == "InfiniDepth_DC":
537
+ assert depth_path is not None and os.path.exists(depth_path), \
538
+ "InfiniDepth_DC requires a valid input depth map for depth completion."
539
+
540
+ _report_stage(stage_callback, "gpu:resolving_depth")
541
+ gt_depth, prompt_depth, gt_depth_mask, prompt_mask, depth_source_label, moge2_pretrained, moge2_intrinsics = \
542
+ _resolve_depth_inputs(depth_path=depth_path, input_size=input_size, image=image, device=device)
543
+ if _debug:
544
+ torch.cuda.synchronize()
545
+ Log.info(f"[GPU-DEBUG] depth_resolved: GPU mem allocated={torch.cuda.memory_allocated(device) / 1e6:.1f}MB")
546
+ _report_stage(stage_callback, f"gpu:depth_resolved source={depth_source_label}")
547
+ Log.info(f"Depth source resolved: {depth_source_label}")
548
+
549
+ gt = depth_to_disparity(gt_depth)
550
+ prompt = depth_to_disparity(prompt_depth)
551
+ prompt_mask = prompt > 0
552
+
553
+ ckpt_path = resolve_checkpoint_path(model_type)
554
+ model_cache = model_cache or ModelCache()
555
+ model = model_cache.get(model_type=model_type, model_path=ckpt_path)
556
+ if _debug:
557
+ torch.cuda.synchronize()
558
+ Log.info(f"[GPU-DEBUG] model_loaded: GPU mem allocated={torch.cuda.memory_allocated(device) / 1e6:.1f}MB")
559
+ _report_stage(stage_callback, "gpu:model_loaded")
560
+
561
+ query_2d_uniform_coord = make_2d_uniform_coord((h_out, w_out)).unsqueeze(0).to(device)
562
+ _report_stage(stage_callback, "gpu:inference_started")
563
+ pred_depth, _ = model.inference(
564
+ image=image, query_coord=query_2d_uniform_coord,
565
+ gt_depth=gt, gt_depth_mask=gt_depth_mask,
566
+ prompt_depth=prompt, prompt_mask=prompt_mask,
567
+ )
568
+ if _debug:
569
+ torch.cuda.synchronize()
570
+ Log.info(f"[GPU-DEBUG] inference_finished: GPU mem allocated={torch.cuda.memory_allocated(device) / 1e6:.1f}MB")
571
+ _report_stage(stage_callback, "gpu:inference_finished")
572
+ Log.info(f"Model inference completed: output_size={h_out}x{w_out}")
573
+
574
+ pred_depthmap = pred_depth.permute(0, 2, 1).reshape(1, 1, h_out, w_out)
575
+
576
+ fx, fy, cx, cy, intrinsics_source_label = resolve_camera_intrinsics_for_inference(
577
+ fx_org=fx_org, fy_org=fy_org, cx_org=cx_org, cy_org=cy_org,
578
+ org_h=org_h, org_w=org_w, image=image,
579
+ moge2_pretrained=moge2_pretrained, moge2_intrinsics=moge2_intrinsics,
580
+ )
581
+ Log.info(f"Camera intrinsics source: {intrinsics_source_label}")
582
+ fx_out, fy_out, cx_out, cy_out, _ = build_scaled_intrinsics_matrix(
583
+ fx_org=fx, fy_org=fy, cx_org=cx, cy_org=cy,
584
+ org_h=org_h, org_w=org_w, h=h_in, w=w_in, device=device,
585
+ )
586
+
587
+ # Transfer all GPU tensors to CPU numpy before returning
588
+ _report_stage(stage_callback, "gpu:transferring_to_cpu")
589
+ result = GPUInferenceResult(
590
+ pred_depthmap_np=pred_depthmap[0, 0].detach().cpu().numpy().astype(np.float32),
591
+ query_coord_np=query_2d_uniform_coord.detach().cpu().numpy().astype(np.float32),
592
+ pred_depth_np=pred_depth.detach().cpu().numpy().astype(np.float32),
593
+ image_tensor_np=image.detach().cpu().numpy().astype(np.float32),
594
+ depth_source_label=depth_source_label,
595
+ intrinsics_source_label=intrinsics_source_label,
596
+ h_out=h_out, w_out=w_out,
597
+ org_h=org_h, org_w=org_w,
598
+ fx_out=float(fx_out), fy_out=float(fy_out),
599
+ cx_out=float(cx_out), cy_out=float(cy_out),
600
+ )
601
+ _report_stage(stage_callback, "gpu:complete")
602
+ return result
603
+
604
+
605
+ def postprocess_gpu_result(
606
+ gpu_result: GPUInferenceResult,
607
+ max_points_preview: int,
608
+ stage_callback: Optional[Callable[[str], None]] = None,
609
+ ) -> DemoResult:
610
+ """CPU-only post-processing: colorize depth, build point cloud, save artifacts.
611
+
612
+ This does not require a GPU and should be called *outside* the ``@spaces.GPU``
613
+ decorated function to avoid consuming ZeroGPU quota.
614
+ """
615
+ _report_stage(stage_callback, "post:start")
616
+
617
+ # Colorize depth (CPU tensors are fine here)
618
+ pred_depthmap_t = torch.from_numpy(gpu_result.pred_depthmap_np).unsqueeze(0).unsqueeze(0)
619
+ depth_vis = _colorize_predicted_depth(pred_depthmap_t)
620
+ _report_stage(stage_callback, "post:depth_colorized")
621
+
622
+ # Save depth numpy
623
+ output_dir = tempfile.mkdtemp(prefix="infinidepth_post_")
624
+ depth_npy_path = os.path.join(output_dir, "depth.npy")
625
+ np.save(depth_npy_path, gpu_result.pred_depthmap_np)
626
+
627
+ # Build and save point cloud (open3d, CPU-only)
628
+ ply_path = os.path.join(output_dir, "point_cloud.ply")
629
+ query_coord_t = torch.from_numpy(gpu_result.query_coord_np)
630
+ pred_depth_t = torch.from_numpy(gpu_result.pred_depth_np)
631
+ image_t = torch.from_numpy(gpu_result.image_tensor_np)
632
+
633
+ xyz, rgb = _build_and_save_point_cloud(
634
+ query_coord=query_coord_t,
635
+ pred_depth=pred_depth_t,
636
+ rgb_image=image_t,
637
+ fx=gpu_result.fx_out, fy=gpu_result.fy_out,
638
+ cx=gpu_result.cx_out, cy=gpu_result.cy_out,
639
+ output_path=ply_path,
640
+ max_points_preview=int(max_points_preview),
641
+ )
642
+ _report_stage(stage_callback, "post:pointcloud_saved")
643
+ Log.info(
644
+ f"Post-processing done: depth_npy={depth_npy_path}, ply={ply_path}, preview_points={xyz.shape[0]}"
645
+ )
646
+
647
+ return DemoResult(
648
+ depth_vis=depth_vis,
649
+ depth_npy_path=depth_npy_path,
650
+ ply_path=ply_path,
651
+ xyz=xyz,
652
+ rgb=rgb,
653
+ depth_source_label=gpu_result.depth_source_label,
654
+ output_size=(gpu_result.h_out, gpu_result.w_out),
655
+ intrinsics_source_label=gpu_result.intrinsics_source_label,
656
+ used_default_intrinsics=gpu_result.intrinsics_source_label == "default",
657
  )
InfiniDepth/utils/inference_utils.py CHANGED
@@ -9,7 +9,10 @@ import torch.nn.functional as F
9
  import open3d as o3d
10
 
11
  from .io_utils import load_depth
12
- from .moge_utils import estimate_metric_depth_with_moge2
 
 
 
13
  from .vis_utils import build_sky_model, run_skyseg
14
 
15
 
@@ -37,15 +40,23 @@ def resolve_camera_intrinsics(
37
  cy_org: Optional[float],
38
  org_h: int,
39
  org_w: int,
 
40
  ) -> tuple[float, float, float, float]:
41
  default_focal = float(max(org_h, org_w))
42
  default_cx = float(org_w) / 2.0
43
  default_cy = float(org_h) / 2.0
44
 
45
- fx = float(fx_org) if fx_org is not None else default_focal
46
- fy = float(fy_org) if fy_org is not None else default_focal
47
- cx = float(cx_org) if cx_org is not None else default_cx
48
- cy = float(cy_org) if cy_org is not None else default_cy
 
 
 
 
 
 
 
49
  return fx, fy, cx, cy
50
 
51
 
@@ -100,7 +111,7 @@ def prepare_metric_depth_inputs(
100
  moge2_pretrained: str,
101
  depth_load_kwargs: Optional[dict] = None,
102
  moge2_kwargs: Optional[dict] = None,
103
- ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, bool]:
104
  depth_load_kwargs = depth_load_kwargs or {}
105
  moge2_kwargs = moge2_kwargs or {}
106
 
@@ -113,9 +124,9 @@ def prepare_metric_depth_inputs(
113
  gt_depth = gt_depth.to(device)
114
  prompt_depth = prompt_depth.to(device)
115
  gt_depth_mask = gt_depth_mask.to(device)
116
- return gt_depth, prompt_depth, gt_depth_mask, True
117
 
118
- pred_depth, gt_depth_mask = estimate_metric_depth_with_moge2(
119
  image=image,
120
  pretrained_model_name_or_path=moge2_pretrained,
121
  **moge2_kwargs,
@@ -123,7 +134,59 @@ def prepare_metric_depth_inputs(
123
  gt_depth = pred_depth.clone().to(device)
124
  prompt_depth = pred_depth.clone().to(device)
125
  gt_depth_mask = gt_depth_mask.to(device)
126
- return gt_depth, prompt_depth, gt_depth_mask, False
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
127
 
128
 
129
  def build_scaled_intrinsics_matrix(
@@ -298,4 +361,4 @@ def build_camera_matrices(
298
  device=device,
299
  ).unsqueeze(0).expand(batch, -1, -1)
300
  extrinsics = torch.eye(4, dtype=torch.float32, device=device).unsqueeze(0).expand(batch, -1, -1)
301
- return fx, fy, cx, cy, intrinsics, extrinsics
 
9
  import open3d as o3d
10
 
11
  from .io_utils import load_depth
12
+ from .moge_utils import (
13
+ estimate_camera_intrinsics_with_moge2,
14
+ estimate_metric_depth_and_intrinsics_with_moge2,
15
+ )
16
  from .vis_utils import build_sky_model, run_skyseg
17
 
18
 
 
40
  cy_org: Optional[float],
41
  org_h: int,
42
  org_w: int,
43
+ fallback_intrinsics: Optional[tuple[float, float, float, float]] = None,
44
  ) -> tuple[float, float, float, float]:
45
  default_focal = float(max(org_h, org_w))
46
  default_cx = float(org_w) / 2.0
47
  default_cy = float(org_h) / 2.0
48
 
49
+ fallback_fx, fallback_fy, fallback_cx, fallback_cy = fallback_intrinsics or (
50
+ default_focal,
51
+ default_focal,
52
+ default_cx,
53
+ default_cy,
54
+ )
55
+
56
+ fx = float(fx_org) if fx_org is not None else float(fallback_fx)
57
+ fy = float(fy_org) if fy_org is not None else float(fallback_fy)
58
+ cx = float(cx_org) if cx_org is not None else float(fallback_cx)
59
+ cy = float(cy_org) if cy_org is not None else float(fallback_cy)
60
  return fx, fy, cx, cy
61
 
62
 
 
111
  moge2_pretrained: str,
112
  depth_load_kwargs: Optional[dict] = None,
113
  moge2_kwargs: Optional[dict] = None,
114
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, bool, Optional[tuple[float, float, float, float]]]:
115
  depth_load_kwargs = depth_load_kwargs or {}
116
  moge2_kwargs = moge2_kwargs or {}
117
 
 
124
  gt_depth = gt_depth.to(device)
125
  prompt_depth = prompt_depth.to(device)
126
  gt_depth_mask = gt_depth_mask.to(device)
127
+ return gt_depth, prompt_depth, gt_depth_mask, True, None
128
 
129
+ pred_depth, gt_depth_mask, moge2_intrinsics = estimate_metric_depth_and_intrinsics_with_moge2(
130
  image=image,
131
  pretrained_model_name_or_path=moge2_pretrained,
132
  **moge2_kwargs,
 
134
  gt_depth = pred_depth.clone().to(device)
135
  prompt_depth = pred_depth.clone().to(device)
136
  gt_depth_mask = gt_depth_mask.to(device)
137
+ return gt_depth, prompt_depth, gt_depth_mask, False, moge2_intrinsics
138
+
139
+
140
+ def resolve_camera_intrinsics_for_inference(
141
+ fx_org: Optional[float],
142
+ fy_org: Optional[float],
143
+ cx_org: Optional[float],
144
+ cy_org: Optional[float],
145
+ org_h: int,
146
+ org_w: int,
147
+ image: torch.Tensor,
148
+ moge2_pretrained: str,
149
+ moge2_intrinsics: Optional[tuple[float, float, float, float]] = None,
150
+ ) -> tuple[float, float, float, float, str]:
151
+ intrinsics_source = "custom"
152
+ fallback_intrinsics = None
153
+
154
+ if has_missing_intrinsics(fx_org, fy_org, cx_org, cy_org):
155
+ if moge2_intrinsics is None:
156
+ try:
157
+ moge2_intrinsics = estimate_camera_intrinsics_with_moge2(
158
+ image=image,
159
+ pretrained_model_name_or_path=moge2_pretrained,
160
+ )
161
+ except Exception as exc:
162
+ print(f"[Warning] Failed to estimate intrinsics with MoGe-2: {exc}")
163
+
164
+ if moge2_intrinsics is not None:
165
+ _, _, h, w = image.shape
166
+ fallback_intrinsics = scale_intrinsics(
167
+ fx=moge2_intrinsics[0],
168
+ fy=moge2_intrinsics[1],
169
+ cx=moge2_intrinsics[2],
170
+ cy=moge2_intrinsics[3],
171
+ org_h=h,
172
+ org_w=w,
173
+ h=org_h,
174
+ w=org_w,
175
+ )
176
+ intrinsics_source = "MoGe-2 estimate"
177
+ else:
178
+ intrinsics_source = "default"
179
+
180
+ fx, fy, cx, cy = resolve_camera_intrinsics(
181
+ fx_org=fx_org,
182
+ fy_org=fy_org,
183
+ cx_org=cx_org,
184
+ cy_org=cy_org,
185
+ org_h=org_h,
186
+ org_w=org_w,
187
+ fallback_intrinsics=fallback_intrinsics,
188
+ )
189
+ return fx, fy, cx, cy, intrinsics_source
190
 
191
 
192
  def build_scaled_intrinsics_matrix(
 
361
  device=device,
362
  ).unsqueeze(0).expand(batch, -1, -1)
363
  extrinsics = torch.eye(4, dtype=torch.float32, device=device).unsqueeze(0).expand(batch, -1, -1)
364
+ return fx, fy, cx, cy, intrinsics, extrinsics
InfiniDepth/utils/moge_utils.py CHANGED
@@ -35,19 +35,40 @@ def _squeeze_hw(tensor: torch.Tensor, name: str) -> torch.Tensor:
35
  raise ValueError(f"Unexpected {name} shape from MoGe-2: {tuple(tensor.shape)}")
36
 
37
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
38
  @torch.no_grad()
39
- def estimate_metric_depth_with_moge2(
40
  image: torch.Tensor,
41
  pretrained_model_name_or_path: str = "Ruicheng/moge-2-vitl-normal",
42
- ) -> tuple[torch.Tensor, torch.Tensor]:
43
- """Run MoGe-2 and return dense pred depth + valid mask in [1,1,H,W]."""
44
  if image.ndim != 4 or image.shape[0] != 1 or image.shape[1] != 3:
45
  raise ValueError(f"Expected image shape [1,3,H,W], got {tuple(image.shape)}")
46
 
47
  device = image.device
48
  model = _get_moge2_model(pretrained_model_name_or_path, device)
49
 
50
- output = model.infer(image[0],apply_mask=True)
51
  if "depth" not in output:
52
  raise KeyError("MoGe-2 output missing key 'depth'.")
53
 
@@ -63,4 +84,42 @@ def estimate_metric_depth_with_moge2(
63
  valid_mask = mask_hw & torch.isfinite(depth_hw) & (depth_hw > 0)
64
  pred_depth = depth_hw * valid_mask.to(depth_hw.dtype)
65
 
66
- return pred_depth.unsqueeze(0).unsqueeze(0), valid_mask.to(torch.float32).unsqueeze(0).unsqueeze(0)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
  raise ValueError(f"Unexpected {name} shape from MoGe-2: {tuple(tensor.shape)}")
36
 
37
 
38
+ def _squeeze_33(tensor: torch.Tensor, name: str) -> torch.Tensor:
39
+ if tensor.ndim == 2 and tensor.shape == (3, 3):
40
+ return tensor
41
+ if tensor.ndim == 3 and tensor.shape[0] == 1 and tensor.shape[1:] == (3, 3):
42
+ return tensor[0]
43
+ raise ValueError(f"Unexpected {name} shape from MoGe-2: {tuple(tensor.shape)}")
44
+
45
+
46
+ def _normalized_intrinsics_to_pixel_intrinsics(
47
+ intrinsics: torch.Tensor,
48
+ height: int,
49
+ width: int,
50
+ ) -> tuple[float, float, float, float]:
51
+ intrinsics = _squeeze_33(intrinsics, "intrinsics")
52
+ fx = float(intrinsics[0, 0].item() * width)
53
+ fy = float(intrinsics[1, 1].item() * height)
54
+ cx = float(intrinsics[0, 2].item() * width - 0.5)
55
+ cy = float(intrinsics[1, 2].item() * height - 0.5)
56
+ return fx, fy, cx, cy
57
+
58
+
59
  @torch.no_grad()
60
+ def estimate_metric_depth_and_intrinsics_with_moge2(
61
  image: torch.Tensor,
62
  pretrained_model_name_or_path: str = "Ruicheng/moge-2-vitl-normal",
63
+ ) -> tuple[torch.Tensor, torch.Tensor, Optional[tuple[float, float, float, float]]]:
64
+ """Run MoGe-2 and return dense pred depth, valid mask, and optional pixel intrinsics."""
65
  if image.ndim != 4 or image.shape[0] != 1 or image.shape[1] != 3:
66
  raise ValueError(f"Expected image shape [1,3,H,W], got {tuple(image.shape)}")
67
 
68
  device = image.device
69
  model = _get_moge2_model(pretrained_model_name_or_path, device)
70
 
71
+ output = model.infer(image[0], apply_mask=True)
72
  if "depth" not in output:
73
  raise KeyError("MoGe-2 output missing key 'depth'.")
74
 
 
84
  valid_mask = mask_hw & torch.isfinite(depth_hw) & (depth_hw > 0)
85
  pred_depth = depth_hw * valid_mask.to(depth_hw.dtype)
86
 
87
+ moge2_intrinsics = None
88
+ output_intrinsics = output.get("intrinsics")
89
+ if output_intrinsics is not None:
90
+ height, width = image.shape[-2:]
91
+ moge2_intrinsics = _normalized_intrinsics_to_pixel_intrinsics(
92
+ output_intrinsics.to(device=device, dtype=torch.float32),
93
+ height=height,
94
+ width=width,
95
+ )
96
+
97
+ return (
98
+ pred_depth.unsqueeze(0).unsqueeze(0),
99
+ valid_mask.to(torch.float32).unsqueeze(0).unsqueeze(0),
100
+ moge2_intrinsics,
101
+ )
102
+
103
+
104
+ @torch.no_grad()
105
+ def estimate_metric_depth_with_moge2(
106
+ image: torch.Tensor,
107
+ pretrained_model_name_or_path: str = "Ruicheng/moge-2-vitl-normal",
108
+ ) -> tuple[torch.Tensor, torch.Tensor]:
109
+ pred_depth, valid_mask, _ = estimate_metric_depth_and_intrinsics_with_moge2(
110
+ image=image,
111
+ pretrained_model_name_or_path=pretrained_model_name_or_path,
112
+ )
113
+ return pred_depth, valid_mask
114
+
115
+
116
+ @torch.no_grad()
117
+ def estimate_camera_intrinsics_with_moge2(
118
+ image: torch.Tensor,
119
+ pretrained_model_name_or_path: str = "Ruicheng/moge-2-vitl-normal",
120
+ ) -> Optional[tuple[float, float, float, float]]:
121
+ _, _, intrinsics = estimate_metric_depth_and_intrinsics_with_moge2(
122
+ image=image,
123
+ pretrained_model_name_or_path=pretrained_model_name_or_path,
124
+ )
125
+ return intrinsics
app.py CHANGED
@@ -23,7 +23,7 @@ import gradio as gr
23
  import numpy as np
24
  from PIL import Image
25
 
26
- from InfiniDepth.utils.hf_demo_utils import ModelCache, prepare_runtime_assets, run_single_image_demo
27
  from InfiniDepth.utils.logger import Log
28
 
29
  try:
@@ -231,21 +231,22 @@ def _export_glb_from_points(xyz: np.ndarray, rgb: np.ndarray, output_path: Path)
231
  cloud.export(output_path.as_posix())
232
 
233
 
234
- def _run_demo_impl(
 
235
  image: np.ndarray,
236
  depth_file,
237
  model_type: str,
238
  input_size: str,
239
  output_resolution_mode: str,
240
  upsample_ratio: int,
241
- max_points_preview: int,
242
  fx_org: Optional[float],
243
  fy_org: Optional[float],
244
  cx_org: Optional[float],
245
  cy_org: Optional[float],
246
  trace_path: str,
247
  ):
248
- _append_trace(trace_path, "worker:entered _run_demo_impl")
 
249
  if image is None:
250
  raise ValueError("Input RGB image is required")
251
 
@@ -253,15 +254,13 @@ def _run_demo_impl(
253
  if depth_file is not None:
254
  depth_path = depth_file if isinstance(depth_file, str) else depth_file.name
255
 
256
- _append_trace(trace_path, f"worker:run_single_image_demo depth_path={depth_path}")
257
- result = run_single_image_demo(
258
  image_np=image,
259
  depth_path=depth_path,
260
  model_type=model_type,
261
  input_size_text=input_size,
262
  output_resolution_mode=output_resolution_mode,
263
  upsample_ratio=int(upsample_ratio),
264
- max_points_preview=int(max_points_preview),
265
  fx_org=_none_if_invalid(fx_org),
266
  fy_org=_none_if_invalid(fy_org),
267
  cx_org=_none_if_invalid(cx_org),
@@ -269,68 +268,6 @@ def _run_demo_impl(
269
  model_cache=MODEL_CACHE,
270
  stage_callback=lambda stage: _append_trace(trace_path, stage),
271
  )
272
- _append_trace(trace_path, "worker:prepare_output_dir")
273
- output_dir = _prepare_output_dir()
274
- glb_path = output_dir / "pointcloud.glb"
275
- ply_path = output_dir / "pointcloud.ply"
276
- depth_vis_path = output_dir / "depth_colorized.png"
277
-
278
- _append_trace(trace_path, "worker:export_glb")
279
- _export_glb_from_points(result.xyz, result.rgb, glb_path)
280
- if os.path.exists(result.ply_path):
281
- shutil.copy2(result.ply_path, ply_path)
282
- depth_vis_uint8 = result.depth_vis if result.depth_vis.dtype == np.uint8 else result.depth_vis.astype(np.uint8)
283
- Image.fromarray(depth_vis_uint8).save(depth_vis_path)
284
-
285
- depth_npy_path = output_dir / "depth.npy"
286
- if os.path.exists(result.depth_npy_path):
287
- shutil.copy2(result.depth_npy_path, depth_npy_path)
288
-
289
- download_files = [glb_path.as_posix(), depth_vis_path.as_posix()]
290
- if depth_npy_path.exists():
291
- download_files.append(depth_npy_path.as_posix())
292
- if ply_path.exists():
293
- download_files.append(ply_path.as_posix())
294
-
295
- status = (
296
- f"Done. Output depth: {result.output_size[0]}x{result.output_size[1]}. "
297
- f"Depth source: {result.depth_source_label}. "
298
- f"Preview points: {result.xyz.shape[0]}. "
299
- f"Camera intrinsics: {'default' if result.used_default_intrinsics else 'custom'}."
300
- )
301
- _append_trace(trace_path, "worker:completed")
302
- return result.depth_vis, glb_path.as_posix(), download_files, status
303
-
304
-
305
- @spaces.GPU(duration=180)
306
- def run_demo_gpu(
307
- image: np.ndarray,
308
- depth_file,
309
- model_type: str,
310
- input_size: str,
311
- output_resolution_mode: str,
312
- upsample_ratio: int,
313
- max_points_preview: int,
314
- fx_org: Optional[float],
315
- fy_org: Optional[float],
316
- cx_org: Optional[float],
317
- cy_org: Optional[float],
318
- trace_path: str,
319
- ):
320
- return _run_demo_impl(
321
- image=image,
322
- depth_file=depth_file,
323
- model_type=model_type,
324
- input_size=input_size,
325
- output_resolution_mode=output_resolution_mode,
326
- upsample_ratio=upsample_ratio,
327
- max_points_preview=max_points_preview,
328
- fx_org=fx_org,
329
- fy_org=fy_org,
330
- cx_org=cx_org,
331
- cy_org=cy_org,
332
- trace_path=trace_path,
333
- )
334
 
335
 
336
  def run_demo(
@@ -360,32 +297,88 @@ def run_demo(
360
  f"depth_path={depth_path}, image_shape={image_shape}"
361
  )
362
  try:
363
- result = run_demo_gpu(
 
364
  image=image,
365
  depth_file=depth_file,
366
  model_type=model_type,
367
  input_size=input_size,
368
  output_resolution_mode=output_resolution_mode,
369
  upsample_ratio=upsample_ratio,
370
- max_points_preview=max_points_preview,
371
  fx_org=fx_org,
372
  fy_org=fy_org,
373
  cx_org=cx_org,
374
  cy_org=cy_org,
375
  trace_path=trace_path,
376
  )
377
- status_text = result[3] if isinstance(result, tuple) and len(result) == 4 else "Done."
378
- Log.info(f"[{request_id}] run_demo success: {status_text}")
379
- return result
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
380
  except Exception as exc:
381
  error_trace = traceback.format_exc()
382
  trace_summary = _read_trace(trace_path)
383
  Log.exception(f"[{request_id}] run_demo failed")
384
- error_message = f"Error [{request_id}]: {exc}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
385
  if trace_summary:
386
  error_message = f"{error_message}\n\nLast worker stages:\n{trace_summary}"
387
  if os.getenv("INFINIDEPTH_SHOW_TRACEBACK", "0") == "1":
388
  error_message = f"{error_message}\n\n{error_trace}"
 
 
389
  return None, None, [], error_message
390
 
391
 
@@ -449,6 +442,7 @@ with gr.Blocks(title="InfiniDepth Demo", theme=gr.themes.Soft(), css=CUSTOM_CSS,
449
  run_button = gr.Button("Generate Depth + 3D", variant="primary", elem_id="run-btn")
450
  gr.Markdown(
451
  "Tips: when a depth map is uploaded it will be used automatically, otherwise the demo falls back to MoGe-2. "
 
452
  "Use lower preview points for faster 3D interaction."
453
  )
454
 
 
23
  import numpy as np
24
  from PIL import Image
25
 
26
+ from InfiniDepth.utils.hf_demo_utils import ModelCache, prepare_runtime_assets, run_gpu_inference, postprocess_gpu_result
27
  from InfiniDepth.utils.logger import Log
28
 
29
  try:
 
231
  cloud.export(output_path.as_posix())
232
 
233
 
234
+ @spaces.GPU(duration=120)
235
+ def run_demo_gpu(
236
  image: np.ndarray,
237
  depth_file,
238
  model_type: str,
239
  input_size: str,
240
  output_resolution_mode: str,
241
  upsample_ratio: int,
 
242
  fx_org: Optional[float],
243
  fy_org: Optional[float],
244
  cx_org: Optional[float],
245
  cy_org: Optional[float],
246
  trace_path: str,
247
  ):
248
+ """GPU-only inference. Returns a GPUInferenceResult with all data on CPU."""
249
+ _append_trace(trace_path, "worker:entered run_demo_gpu")
250
  if image is None:
251
  raise ValueError("Input RGB image is required")
252
 
 
254
  if depth_file is not None:
255
  depth_path = depth_file if isinstance(depth_file, str) else depth_file.name
256
 
257
+ return run_gpu_inference(
 
258
  image_np=image,
259
  depth_path=depth_path,
260
  model_type=model_type,
261
  input_size_text=input_size,
262
  output_resolution_mode=output_resolution_mode,
263
  upsample_ratio=int(upsample_ratio),
 
264
  fx_org=_none_if_invalid(fx_org),
265
  fy_org=_none_if_invalid(fy_org),
266
  cx_org=_none_if_invalid(cx_org),
 
268
  model_cache=MODEL_CACHE,
269
  stage_callback=lambda stage: _append_trace(trace_path, stage),
270
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
271
 
272
 
273
  def run_demo(
 
297
  f"depth_path={depth_path}, image_shape={image_shape}"
298
  )
299
  try:
300
+ # --- GPU-only inference (consumes ZeroGPU quota) ---
301
+ gpu_result = run_demo_gpu(
302
  image=image,
303
  depth_file=depth_file,
304
  model_type=model_type,
305
  input_size=input_size,
306
  output_resolution_mode=output_resolution_mode,
307
  upsample_ratio=upsample_ratio,
 
308
  fx_org=fx_org,
309
  fy_org=fy_org,
310
  cx_org=cx_org,
311
  cy_org=cy_org,
312
  trace_path=trace_path,
313
  )
314
+ _append_trace(trace_path, "ui:gpu_done, starting cpu postprocess")
315
+
316
+ # --- CPU post-processing (no GPU quota consumed) ---
317
+ result = postprocess_gpu_result(
318
+ gpu_result=gpu_result,
319
+ max_points_preview=int(max_points_preview),
320
+ stage_callback=lambda stage: _append_trace(trace_path, stage),
321
+ )
322
+ _append_trace(trace_path, "ui:postprocess_done, exporting files")
323
+
324
+ # --- File export ---
325
+ output_dir = _prepare_output_dir()
326
+ glb_path = output_dir / "pointcloud.glb"
327
+ ply_path = output_dir / "pointcloud.ply"
328
+ depth_vis_path = output_dir / "depth_colorized.png"
329
+
330
+ _export_glb_from_points(result.xyz, result.rgb, glb_path)
331
+ if os.path.exists(result.ply_path):
332
+ shutil.copy2(result.ply_path, ply_path)
333
+ depth_vis_uint8 = result.depth_vis if result.depth_vis.dtype == np.uint8 else result.depth_vis.astype(np.uint8)
334
+ Image.fromarray(depth_vis_uint8).save(depth_vis_path)
335
+
336
+ depth_npy_path = output_dir / "depth.npy"
337
+ if os.path.exists(result.depth_npy_path):
338
+ shutil.copy2(result.depth_npy_path, depth_npy_path)
339
+
340
+ download_files = [glb_path.as_posix(), depth_vis_path.as_posix()]
341
+ if depth_npy_path.exists():
342
+ download_files.append(depth_npy_path.as_posix())
343
+ if ply_path.exists():
344
+ download_files.append(ply_path.as_posix())
345
+
346
+ status = (
347
+ f"Done. Output depth: {result.output_size[0]}x{result.output_size[1]}. "
348
+ f"Depth source: {result.depth_source_label}. "
349
+ f"Preview points: {result.xyz.shape[0]}. "
350
+ f"Camera intrinsics: {result.intrinsics_source_label}."
351
+ )
352
+ _append_trace(trace_path, "ui:completed")
353
+ Log.info(f"[{request_id}] run_demo success: {status}")
354
+ return result.depth_vis, glb_path.as_posix(), download_files, status
355
  except Exception as exc:
356
  error_trace = traceback.format_exc()
357
  trace_summary = _read_trace(trace_path)
358
  Log.exception(f"[{request_id}] run_demo failed")
359
+
360
+ # Classify the error for clearer diagnostics
361
+ exc_type = type(exc).__name__
362
+ exc_module = type(exc).__module__ or ""
363
+ is_zerogpu_error = "spaces" in exc_module or "ZeroGPU" in str(exc) or "GPU task aborted" in str(exc)
364
+ if is_zerogpu_error:
365
+ error_message = (
366
+ f"[{request_id}] ZeroGPU error: {exc}\n\n"
367
+ "This is a HuggingFace ZeroGPU scheduling issue, not an inference bug.\n"
368
+ "Possible causes:\n"
369
+ " - GPU quota exhausted (wait for quota to reset)\n"
370
+ " - GPU task was preempted/aborted (try again)\n"
371
+ " - duration too high for remaining quota"
372
+ )
373
+ else:
374
+ error_message = f"Error [{request_id}] ({exc_type}): {exc}"
375
+
376
  if trace_summary:
377
  error_message = f"{error_message}\n\nLast worker stages:\n{trace_summary}"
378
  if os.getenv("INFINIDEPTH_SHOW_TRACEBACK", "0") == "1":
379
  error_message = f"{error_message}\n\n{error_trace}"
380
+ # Always log full traceback to server logs (visible in HF Space Logs tab)
381
+ Log.error(f"[{request_id}] Full traceback:\n{error_trace}")
382
  return None, None, [], error_message
383
 
384
 
 
442
  run_button = gr.Button("Generate Depth + 3D", variant="primary", elem_id="run-btn")
443
  gr.Markdown(
444
  "Tips: when a depth map is uploaded it will be used automatically, otherwise the demo falls back to MoGe-2. "
445
+ "If camera intrinsics are missing, the demo first tries MoGe-2 estimates before image-size defaults. "
446
  "Use lower preview points for faster 3D interaction."
447
  )
448