Spaces:
Paused
Paused
update
Browse files- InfiniDepth/utils/hf_demo_utils.py +227 -8
- InfiniDepth/utils/inference_utils.py +73 -10
- InfiniDepth/utils/moge_utils.py +64 -5
- app.py +69 -75
InfiniDepth/utils/hf_demo_utils.py
CHANGED
|
@@ -12,9 +12,8 @@ from huggingface_hub import hf_hub_download
|
|
| 12 |
|
| 13 |
from .inference_utils import (
|
| 14 |
build_scaled_intrinsics_matrix,
|
| 15 |
-
has_missing_intrinsics,
|
| 16 |
prepare_metric_depth_inputs,
|
| 17 |
-
|
| 18 |
resolve_output_size_from_mode,
|
| 19 |
)
|
| 20 |
from .io_utils import depth2pcd, depth_to_disparity
|
|
@@ -143,9 +142,29 @@ class DemoResult:
|
|
| 143 |
rgb: np.ndarray
|
| 144 |
depth_source_label: str
|
| 145 |
output_size: tuple[int, int]
|
|
|
|
| 146 |
used_default_intrinsics: bool
|
| 147 |
|
| 148 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 149 |
class ModelCache:
|
| 150 |
def __init__(self):
|
| 151 |
self._cache: dict[tuple[str, str], Any] = {}
|
|
@@ -207,10 +226,18 @@ def _resolve_depth_inputs(
|
|
| 207 |
input_size: tuple[int, int],
|
| 208 |
image: torch.Tensor,
|
| 209 |
device: torch.device,
|
| 210 |
-
) -> tuple[
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
input_depth_path = depth_path if depth_path else None
|
| 212 |
moge2_pretrained = resolve_moge2_pretrained()
|
| 213 |
-
gt_depth, prompt_depth, gt_depth_mask, used_input_depth = prepare_metric_depth_inputs(
|
| 214 |
input_depth_path=input_depth_path,
|
| 215 |
input_size=input_size,
|
| 216 |
image=image,
|
|
@@ -220,7 +247,7 @@ def _resolve_depth_inputs(
|
|
| 220 |
)
|
| 221 |
prompt_mask = prompt_depth > 0
|
| 222 |
depth_source_label = "uploaded depth" if used_input_depth else "MoGe-2 prior"
|
| 223 |
-
return gt_depth, prompt_depth, gt_depth_mask, prompt_mask, depth_source_label
|
| 224 |
|
| 225 |
|
| 226 |
def _colorize_predicted_depth(pred_depthmap: torch.Tensor) -> np.ndarray:
|
|
@@ -357,7 +384,7 @@ def run_single_image_demo(
|
|
| 357 |
if model_type == "InfiniDepth_DC":
|
| 358 |
assert depth_path is not None and os.path.exists(depth_path), "InfiniDepth_DC requires a valid input depth map for depth completion. Please provide --input_depth_path."
|
| 359 |
|
| 360 |
-
gt_depth, prompt_depth, gt_depth_mask, prompt_mask, depth_source_label = _resolve_depth_inputs(
|
| 361 |
depth_path=depth_path,
|
| 362 |
input_size=input_size,
|
| 363 |
image=image,
|
|
@@ -392,7 +419,18 @@ def run_single_image_demo(
|
|
| 392 |
depth_vis = _colorize_predicted_depth(pred_depthmap)
|
| 393 |
_report_stage(stage_callback, "demo:depth_colorized")
|
| 394 |
|
| 395 |
-
fx, fy, cx, cy
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 396 |
fx_out, fy_out, cx_out, cy_out, _ = build_scaled_intrinsics_matrix(
|
| 397 |
fx_org=fx,
|
| 398 |
fy_org=fy,
|
|
@@ -434,5 +472,186 @@ def run_single_image_demo(
|
|
| 434 |
rgb=rgb,
|
| 435 |
depth_source_label=depth_source_label,
|
| 436 |
output_size=(h_out, w_out),
|
| 437 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 438 |
)
|
|
|
|
| 12 |
|
| 13 |
from .inference_utils import (
|
| 14 |
build_scaled_intrinsics_matrix,
|
|
|
|
| 15 |
prepare_metric_depth_inputs,
|
| 16 |
+
resolve_camera_intrinsics_for_inference,
|
| 17 |
resolve_output_size_from_mode,
|
| 18 |
)
|
| 19 |
from .io_utils import depth2pcd, depth_to_disparity
|
|
|
|
| 142 |
rgb: np.ndarray
|
| 143 |
depth_source_label: str
|
| 144 |
output_size: tuple[int, int]
|
| 145 |
+
intrinsics_source_label: str
|
| 146 |
used_default_intrinsics: bool
|
| 147 |
|
| 148 |
|
| 149 |
+
@dataclass
|
| 150 |
+
class GPUInferenceResult:
|
| 151 |
+
"""Raw GPU inference outputs, all transferred to CPU numpy arrays."""
|
| 152 |
+
pred_depthmap_np: np.ndarray
|
| 153 |
+
query_coord_np: np.ndarray
|
| 154 |
+
pred_depth_np: np.ndarray
|
| 155 |
+
image_tensor_np: np.ndarray
|
| 156 |
+
depth_source_label: str
|
| 157 |
+
intrinsics_source_label: str
|
| 158 |
+
h_out: int
|
| 159 |
+
w_out: int
|
| 160 |
+
org_h: int
|
| 161 |
+
org_w: int
|
| 162 |
+
fx_out: float
|
| 163 |
+
fy_out: float
|
| 164 |
+
cx_out: float
|
| 165 |
+
cy_out: float
|
| 166 |
+
|
| 167 |
+
|
| 168 |
class ModelCache:
|
| 169 |
def __init__(self):
|
| 170 |
self._cache: dict[tuple[str, str], Any] = {}
|
|
|
|
| 226 |
input_size: tuple[int, int],
|
| 227 |
image: torch.Tensor,
|
| 228 |
device: torch.device,
|
| 229 |
+
) -> tuple[
|
| 230 |
+
torch.Tensor,
|
| 231 |
+
torch.Tensor,
|
| 232 |
+
torch.Tensor,
|
| 233 |
+
torch.Tensor,
|
| 234 |
+
str,
|
| 235 |
+
str,
|
| 236 |
+
Optional[tuple[float, float, float, float]],
|
| 237 |
+
]:
|
| 238 |
input_depth_path = depth_path if depth_path else None
|
| 239 |
moge2_pretrained = resolve_moge2_pretrained()
|
| 240 |
+
gt_depth, prompt_depth, gt_depth_mask, used_input_depth, moge2_intrinsics = prepare_metric_depth_inputs(
|
| 241 |
input_depth_path=input_depth_path,
|
| 242 |
input_size=input_size,
|
| 243 |
image=image,
|
|
|
|
| 247 |
)
|
| 248 |
prompt_mask = prompt_depth > 0
|
| 249 |
depth_source_label = "uploaded depth" if used_input_depth else "MoGe-2 prior"
|
| 250 |
+
return gt_depth, prompt_depth, gt_depth_mask, prompt_mask, depth_source_label, moge2_pretrained, moge2_intrinsics
|
| 251 |
|
| 252 |
|
| 253 |
def _colorize_predicted_depth(pred_depthmap: torch.Tensor) -> np.ndarray:
|
|
|
|
| 384 |
if model_type == "InfiniDepth_DC":
|
| 385 |
assert depth_path is not None and os.path.exists(depth_path), "InfiniDepth_DC requires a valid input depth map for depth completion. Please provide --input_depth_path."
|
| 386 |
|
| 387 |
+
gt_depth, prompt_depth, gt_depth_mask, prompt_mask, depth_source_label, moge2_pretrained, moge2_intrinsics = _resolve_depth_inputs(
|
| 388 |
depth_path=depth_path,
|
| 389 |
input_size=input_size,
|
| 390 |
image=image,
|
|
|
|
| 419 |
depth_vis = _colorize_predicted_depth(pred_depthmap)
|
| 420 |
_report_stage(stage_callback, "demo:depth_colorized")
|
| 421 |
|
| 422 |
+
fx, fy, cx, cy, intrinsics_source_label = resolve_camera_intrinsics_for_inference(
|
| 423 |
+
fx_org=fx_org,
|
| 424 |
+
fy_org=fy_org,
|
| 425 |
+
cx_org=cx_org,
|
| 426 |
+
cy_org=cy_org,
|
| 427 |
+
org_h=org_h,
|
| 428 |
+
org_w=org_w,
|
| 429 |
+
image=image,
|
| 430 |
+
moge2_pretrained=moge2_pretrained,
|
| 431 |
+
moge2_intrinsics=moge2_intrinsics,
|
| 432 |
+
)
|
| 433 |
+
Log.info(f"Camera intrinsics source resolved: {intrinsics_source_label}")
|
| 434 |
fx_out, fy_out, cx_out, cy_out, _ = build_scaled_intrinsics_matrix(
|
| 435 |
fx_org=fx,
|
| 436 |
fy_org=fy,
|
|
|
|
| 472 |
rgb=rgb,
|
| 473 |
depth_source_label=depth_source_label,
|
| 474 |
output_size=(h_out, w_out),
|
| 475 |
+
intrinsics_source_label=intrinsics_source_label,
|
| 476 |
+
used_default_intrinsics=intrinsics_source_label == "default",
|
| 477 |
+
)
|
| 478 |
+
|
| 479 |
+
|
| 480 |
+
def run_gpu_inference(
|
| 481 |
+
image_np: np.ndarray,
|
| 482 |
+
depth_path: Optional[str],
|
| 483 |
+
model_type: str,
|
| 484 |
+
input_size_text: str,
|
| 485 |
+
output_resolution_mode: str,
|
| 486 |
+
upsample_ratio: int,
|
| 487 |
+
fx_org: Optional[float] = None,
|
| 488 |
+
fy_org: Optional[float] = None,
|
| 489 |
+
cx_org: Optional[float] = None,
|
| 490 |
+
cy_org: Optional[float] = None,
|
| 491 |
+
model_cache: Optional[ModelCache] = None,
|
| 492 |
+
stage_callback: Optional[Callable[[str], None]] = None,
|
| 493 |
+
) -> GPUInferenceResult:
|
| 494 |
+
"""Run only GPU-intensive inference. All outputs are moved to CPU numpy before return.
|
| 495 |
+
|
| 496 |
+
This is designed for HuggingFace ZeroGPU where GPU time is limited: only the actual
|
| 497 |
+
CUDA work (MoGe-2, model inference, intrinsics estimation) runs here. CPU-heavy
|
| 498 |
+
post-processing (colorization, point cloud, file I/O) should happen outside the
|
| 499 |
+
``@spaces.GPU`` decorated caller.
|
| 500 |
+
"""
|
| 501 |
+
image_shape = tuple(int(d) for d in image_np.shape) if image_np is not None else None
|
| 502 |
+
_report_stage(stage_callback, "gpu:start")
|
| 503 |
+
Log.info(
|
| 504 |
+
f"run_gpu_inference: model_type={model_type}, input_size={input_size_text}, "
|
| 505 |
+
f"output_resolution_mode={output_resolution_mode}, upsample_ratio={upsample_ratio}, "
|
| 506 |
+
f"has_depth={bool(depth_path)}, image_shape={image_shape}, "
|
| 507 |
+
f"cuda_available={torch.cuda.is_available()}"
|
| 508 |
+
)
|
| 509 |
+
if not torch.cuda.is_available():
|
| 510 |
+
raise RuntimeError(
|
| 511 |
+
"No CUDA GPU is available. If using Hugging Face ZeroGPU, "
|
| 512 |
+
"decorate the Gradio inference function with @spaces.GPU and enable queue()."
|
| 513 |
+
)
|
| 514 |
+
|
| 515 |
+
input_size = _parse_image_size(input_size_text)
|
| 516 |
+
if upsample_ratio < 1 or upsample_ratio > 8:
|
| 517 |
+
raise ValueError("upsample_ratio must be in [1, 8]")
|
| 518 |
+
output_size = input_size
|
| 519 |
+
device = torch.device("cuda")
|
| 520 |
+
|
| 521 |
+
_debug = os.getenv("INFINIDEPTH_DEBUG_GPU", "0") == "1"
|
| 522 |
+
|
| 523 |
+
image, org_h, org_w = _prepare_image_tensor(image_np, input_size, device)
|
| 524 |
+
_report_stage(stage_callback, "gpu:image_prepared")
|
| 525 |
+
if _debug:
|
| 526 |
+
torch.cuda.synchronize()
|
| 527 |
+
Log.info(f"[GPU-DEBUG] image_prepared: GPU mem allocated={torch.cuda.memory_allocated(device) / 1e6:.1f}MB")
|
| 528 |
+
|
| 529 |
+
h_in, w_in = input_size
|
| 530 |
+
h_out, w_out = resolve_output_size_from_mode(
|
| 531 |
+
output_resolution_mode=output_resolution_mode,
|
| 532 |
+
org_h=org_h, org_w=org_w, h=h_in, w=w_in,
|
| 533 |
+
output_size=output_size, upsample_ratio=upsample_ratio,
|
| 534 |
+
)
|
| 535 |
+
|
| 536 |
+
if model_type == "InfiniDepth_DC":
|
| 537 |
+
assert depth_path is not None and os.path.exists(depth_path), \
|
| 538 |
+
"InfiniDepth_DC requires a valid input depth map for depth completion."
|
| 539 |
+
|
| 540 |
+
_report_stage(stage_callback, "gpu:resolving_depth")
|
| 541 |
+
gt_depth, prompt_depth, gt_depth_mask, prompt_mask, depth_source_label, moge2_pretrained, moge2_intrinsics = \
|
| 542 |
+
_resolve_depth_inputs(depth_path=depth_path, input_size=input_size, image=image, device=device)
|
| 543 |
+
if _debug:
|
| 544 |
+
torch.cuda.synchronize()
|
| 545 |
+
Log.info(f"[GPU-DEBUG] depth_resolved: GPU mem allocated={torch.cuda.memory_allocated(device) / 1e6:.1f}MB")
|
| 546 |
+
_report_stage(stage_callback, f"gpu:depth_resolved source={depth_source_label}")
|
| 547 |
+
Log.info(f"Depth source resolved: {depth_source_label}")
|
| 548 |
+
|
| 549 |
+
gt = depth_to_disparity(gt_depth)
|
| 550 |
+
prompt = depth_to_disparity(prompt_depth)
|
| 551 |
+
prompt_mask = prompt > 0
|
| 552 |
+
|
| 553 |
+
ckpt_path = resolve_checkpoint_path(model_type)
|
| 554 |
+
model_cache = model_cache or ModelCache()
|
| 555 |
+
model = model_cache.get(model_type=model_type, model_path=ckpt_path)
|
| 556 |
+
if _debug:
|
| 557 |
+
torch.cuda.synchronize()
|
| 558 |
+
Log.info(f"[GPU-DEBUG] model_loaded: GPU mem allocated={torch.cuda.memory_allocated(device) / 1e6:.1f}MB")
|
| 559 |
+
_report_stage(stage_callback, "gpu:model_loaded")
|
| 560 |
+
|
| 561 |
+
query_2d_uniform_coord = make_2d_uniform_coord((h_out, w_out)).unsqueeze(0).to(device)
|
| 562 |
+
_report_stage(stage_callback, "gpu:inference_started")
|
| 563 |
+
pred_depth, _ = model.inference(
|
| 564 |
+
image=image, query_coord=query_2d_uniform_coord,
|
| 565 |
+
gt_depth=gt, gt_depth_mask=gt_depth_mask,
|
| 566 |
+
prompt_depth=prompt, prompt_mask=prompt_mask,
|
| 567 |
+
)
|
| 568 |
+
if _debug:
|
| 569 |
+
torch.cuda.synchronize()
|
| 570 |
+
Log.info(f"[GPU-DEBUG] inference_finished: GPU mem allocated={torch.cuda.memory_allocated(device) / 1e6:.1f}MB")
|
| 571 |
+
_report_stage(stage_callback, "gpu:inference_finished")
|
| 572 |
+
Log.info(f"Model inference completed: output_size={h_out}x{w_out}")
|
| 573 |
+
|
| 574 |
+
pred_depthmap = pred_depth.permute(0, 2, 1).reshape(1, 1, h_out, w_out)
|
| 575 |
+
|
| 576 |
+
fx, fy, cx, cy, intrinsics_source_label = resolve_camera_intrinsics_for_inference(
|
| 577 |
+
fx_org=fx_org, fy_org=fy_org, cx_org=cx_org, cy_org=cy_org,
|
| 578 |
+
org_h=org_h, org_w=org_w, image=image,
|
| 579 |
+
moge2_pretrained=moge2_pretrained, moge2_intrinsics=moge2_intrinsics,
|
| 580 |
+
)
|
| 581 |
+
Log.info(f"Camera intrinsics source: {intrinsics_source_label}")
|
| 582 |
+
fx_out, fy_out, cx_out, cy_out, _ = build_scaled_intrinsics_matrix(
|
| 583 |
+
fx_org=fx, fy_org=fy, cx_org=cx, cy_org=cy,
|
| 584 |
+
org_h=org_h, org_w=org_w, h=h_in, w=w_in, device=device,
|
| 585 |
+
)
|
| 586 |
+
|
| 587 |
+
# Transfer all GPU tensors to CPU numpy before returning
|
| 588 |
+
_report_stage(stage_callback, "gpu:transferring_to_cpu")
|
| 589 |
+
result = GPUInferenceResult(
|
| 590 |
+
pred_depthmap_np=pred_depthmap[0, 0].detach().cpu().numpy().astype(np.float32),
|
| 591 |
+
query_coord_np=query_2d_uniform_coord.detach().cpu().numpy().astype(np.float32),
|
| 592 |
+
pred_depth_np=pred_depth.detach().cpu().numpy().astype(np.float32),
|
| 593 |
+
image_tensor_np=image.detach().cpu().numpy().astype(np.float32),
|
| 594 |
+
depth_source_label=depth_source_label,
|
| 595 |
+
intrinsics_source_label=intrinsics_source_label,
|
| 596 |
+
h_out=h_out, w_out=w_out,
|
| 597 |
+
org_h=org_h, org_w=org_w,
|
| 598 |
+
fx_out=float(fx_out), fy_out=float(fy_out),
|
| 599 |
+
cx_out=float(cx_out), cy_out=float(cy_out),
|
| 600 |
+
)
|
| 601 |
+
_report_stage(stage_callback, "gpu:complete")
|
| 602 |
+
return result
|
| 603 |
+
|
| 604 |
+
|
| 605 |
+
def postprocess_gpu_result(
|
| 606 |
+
gpu_result: GPUInferenceResult,
|
| 607 |
+
max_points_preview: int,
|
| 608 |
+
stage_callback: Optional[Callable[[str], None]] = None,
|
| 609 |
+
) -> DemoResult:
|
| 610 |
+
"""CPU-only post-processing: colorize depth, build point cloud, save artifacts.
|
| 611 |
+
|
| 612 |
+
This does not require a GPU and should be called *outside* the ``@spaces.GPU``
|
| 613 |
+
decorated function to avoid consuming ZeroGPU quota.
|
| 614 |
+
"""
|
| 615 |
+
_report_stage(stage_callback, "post:start")
|
| 616 |
+
|
| 617 |
+
# Colorize depth (CPU tensors are fine here)
|
| 618 |
+
pred_depthmap_t = torch.from_numpy(gpu_result.pred_depthmap_np).unsqueeze(0).unsqueeze(0)
|
| 619 |
+
depth_vis = _colorize_predicted_depth(pred_depthmap_t)
|
| 620 |
+
_report_stage(stage_callback, "post:depth_colorized")
|
| 621 |
+
|
| 622 |
+
# Save depth numpy
|
| 623 |
+
output_dir = tempfile.mkdtemp(prefix="infinidepth_post_")
|
| 624 |
+
depth_npy_path = os.path.join(output_dir, "depth.npy")
|
| 625 |
+
np.save(depth_npy_path, gpu_result.pred_depthmap_np)
|
| 626 |
+
|
| 627 |
+
# Build and save point cloud (open3d, CPU-only)
|
| 628 |
+
ply_path = os.path.join(output_dir, "point_cloud.ply")
|
| 629 |
+
query_coord_t = torch.from_numpy(gpu_result.query_coord_np)
|
| 630 |
+
pred_depth_t = torch.from_numpy(gpu_result.pred_depth_np)
|
| 631 |
+
image_t = torch.from_numpy(gpu_result.image_tensor_np)
|
| 632 |
+
|
| 633 |
+
xyz, rgb = _build_and_save_point_cloud(
|
| 634 |
+
query_coord=query_coord_t,
|
| 635 |
+
pred_depth=pred_depth_t,
|
| 636 |
+
rgb_image=image_t,
|
| 637 |
+
fx=gpu_result.fx_out, fy=gpu_result.fy_out,
|
| 638 |
+
cx=gpu_result.cx_out, cy=gpu_result.cy_out,
|
| 639 |
+
output_path=ply_path,
|
| 640 |
+
max_points_preview=int(max_points_preview),
|
| 641 |
+
)
|
| 642 |
+
_report_stage(stage_callback, "post:pointcloud_saved")
|
| 643 |
+
Log.info(
|
| 644 |
+
f"Post-processing done: depth_npy={depth_npy_path}, ply={ply_path}, preview_points={xyz.shape[0]}"
|
| 645 |
+
)
|
| 646 |
+
|
| 647 |
+
return DemoResult(
|
| 648 |
+
depth_vis=depth_vis,
|
| 649 |
+
depth_npy_path=depth_npy_path,
|
| 650 |
+
ply_path=ply_path,
|
| 651 |
+
xyz=xyz,
|
| 652 |
+
rgb=rgb,
|
| 653 |
+
depth_source_label=gpu_result.depth_source_label,
|
| 654 |
+
output_size=(gpu_result.h_out, gpu_result.w_out),
|
| 655 |
+
intrinsics_source_label=gpu_result.intrinsics_source_label,
|
| 656 |
+
used_default_intrinsics=gpu_result.intrinsics_source_label == "default",
|
| 657 |
)
|
InfiniDepth/utils/inference_utils.py
CHANGED
|
@@ -9,7 +9,10 @@ import torch.nn.functional as F
|
|
| 9 |
import open3d as o3d
|
| 10 |
|
| 11 |
from .io_utils import load_depth
|
| 12 |
-
from .moge_utils import
|
|
|
|
|
|
|
|
|
|
| 13 |
from .vis_utils import build_sky_model, run_skyseg
|
| 14 |
|
| 15 |
|
|
@@ -37,15 +40,23 @@ def resolve_camera_intrinsics(
|
|
| 37 |
cy_org: Optional[float],
|
| 38 |
org_h: int,
|
| 39 |
org_w: int,
|
|
|
|
| 40 |
) -> tuple[float, float, float, float]:
|
| 41 |
default_focal = float(max(org_h, org_w))
|
| 42 |
default_cx = float(org_w) / 2.0
|
| 43 |
default_cy = float(org_h) / 2.0
|
| 44 |
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
return fx, fy, cx, cy
|
| 50 |
|
| 51 |
|
|
@@ -100,7 +111,7 @@ def prepare_metric_depth_inputs(
|
|
| 100 |
moge2_pretrained: str,
|
| 101 |
depth_load_kwargs: Optional[dict] = None,
|
| 102 |
moge2_kwargs: Optional[dict] = None,
|
| 103 |
-
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, bool]:
|
| 104 |
depth_load_kwargs = depth_load_kwargs or {}
|
| 105 |
moge2_kwargs = moge2_kwargs or {}
|
| 106 |
|
|
@@ -113,9 +124,9 @@ def prepare_metric_depth_inputs(
|
|
| 113 |
gt_depth = gt_depth.to(device)
|
| 114 |
prompt_depth = prompt_depth.to(device)
|
| 115 |
gt_depth_mask = gt_depth_mask.to(device)
|
| 116 |
-
return gt_depth, prompt_depth, gt_depth_mask, True
|
| 117 |
|
| 118 |
-
pred_depth, gt_depth_mask =
|
| 119 |
image=image,
|
| 120 |
pretrained_model_name_or_path=moge2_pretrained,
|
| 121 |
**moge2_kwargs,
|
|
@@ -123,7 +134,59 @@ def prepare_metric_depth_inputs(
|
|
| 123 |
gt_depth = pred_depth.clone().to(device)
|
| 124 |
prompt_depth = pred_depth.clone().to(device)
|
| 125 |
gt_depth_mask = gt_depth_mask.to(device)
|
| 126 |
-
return gt_depth, prompt_depth, gt_depth_mask, False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
|
| 128 |
|
| 129 |
def build_scaled_intrinsics_matrix(
|
|
@@ -298,4 +361,4 @@ def build_camera_matrices(
|
|
| 298 |
device=device,
|
| 299 |
).unsqueeze(0).expand(batch, -1, -1)
|
| 300 |
extrinsics = torch.eye(4, dtype=torch.float32, device=device).unsqueeze(0).expand(batch, -1, -1)
|
| 301 |
-
return fx, fy, cx, cy, intrinsics, extrinsics
|
|
|
|
| 9 |
import open3d as o3d
|
| 10 |
|
| 11 |
from .io_utils import load_depth
|
| 12 |
+
from .moge_utils import (
|
| 13 |
+
estimate_camera_intrinsics_with_moge2,
|
| 14 |
+
estimate_metric_depth_and_intrinsics_with_moge2,
|
| 15 |
+
)
|
| 16 |
from .vis_utils import build_sky_model, run_skyseg
|
| 17 |
|
| 18 |
|
|
|
|
| 40 |
cy_org: Optional[float],
|
| 41 |
org_h: int,
|
| 42 |
org_w: int,
|
| 43 |
+
fallback_intrinsics: Optional[tuple[float, float, float, float]] = None,
|
| 44 |
) -> tuple[float, float, float, float]:
|
| 45 |
default_focal = float(max(org_h, org_w))
|
| 46 |
default_cx = float(org_w) / 2.0
|
| 47 |
default_cy = float(org_h) / 2.0
|
| 48 |
|
| 49 |
+
fallback_fx, fallback_fy, fallback_cx, fallback_cy = fallback_intrinsics or (
|
| 50 |
+
default_focal,
|
| 51 |
+
default_focal,
|
| 52 |
+
default_cx,
|
| 53 |
+
default_cy,
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
fx = float(fx_org) if fx_org is not None else float(fallback_fx)
|
| 57 |
+
fy = float(fy_org) if fy_org is not None else float(fallback_fy)
|
| 58 |
+
cx = float(cx_org) if cx_org is not None else float(fallback_cx)
|
| 59 |
+
cy = float(cy_org) if cy_org is not None else float(fallback_cy)
|
| 60 |
return fx, fy, cx, cy
|
| 61 |
|
| 62 |
|
|
|
|
| 111 |
moge2_pretrained: str,
|
| 112 |
depth_load_kwargs: Optional[dict] = None,
|
| 113 |
moge2_kwargs: Optional[dict] = None,
|
| 114 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, bool, Optional[tuple[float, float, float, float]]]:
|
| 115 |
depth_load_kwargs = depth_load_kwargs or {}
|
| 116 |
moge2_kwargs = moge2_kwargs or {}
|
| 117 |
|
|
|
|
| 124 |
gt_depth = gt_depth.to(device)
|
| 125 |
prompt_depth = prompt_depth.to(device)
|
| 126 |
gt_depth_mask = gt_depth_mask.to(device)
|
| 127 |
+
return gt_depth, prompt_depth, gt_depth_mask, True, None
|
| 128 |
|
| 129 |
+
pred_depth, gt_depth_mask, moge2_intrinsics = estimate_metric_depth_and_intrinsics_with_moge2(
|
| 130 |
image=image,
|
| 131 |
pretrained_model_name_or_path=moge2_pretrained,
|
| 132 |
**moge2_kwargs,
|
|
|
|
| 134 |
gt_depth = pred_depth.clone().to(device)
|
| 135 |
prompt_depth = pred_depth.clone().to(device)
|
| 136 |
gt_depth_mask = gt_depth_mask.to(device)
|
| 137 |
+
return gt_depth, prompt_depth, gt_depth_mask, False, moge2_intrinsics
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def resolve_camera_intrinsics_for_inference(
|
| 141 |
+
fx_org: Optional[float],
|
| 142 |
+
fy_org: Optional[float],
|
| 143 |
+
cx_org: Optional[float],
|
| 144 |
+
cy_org: Optional[float],
|
| 145 |
+
org_h: int,
|
| 146 |
+
org_w: int,
|
| 147 |
+
image: torch.Tensor,
|
| 148 |
+
moge2_pretrained: str,
|
| 149 |
+
moge2_intrinsics: Optional[tuple[float, float, float, float]] = None,
|
| 150 |
+
) -> tuple[float, float, float, float, str]:
|
| 151 |
+
intrinsics_source = "custom"
|
| 152 |
+
fallback_intrinsics = None
|
| 153 |
+
|
| 154 |
+
if has_missing_intrinsics(fx_org, fy_org, cx_org, cy_org):
|
| 155 |
+
if moge2_intrinsics is None:
|
| 156 |
+
try:
|
| 157 |
+
moge2_intrinsics = estimate_camera_intrinsics_with_moge2(
|
| 158 |
+
image=image,
|
| 159 |
+
pretrained_model_name_or_path=moge2_pretrained,
|
| 160 |
+
)
|
| 161 |
+
except Exception as exc:
|
| 162 |
+
print(f"[Warning] Failed to estimate intrinsics with MoGe-2: {exc}")
|
| 163 |
+
|
| 164 |
+
if moge2_intrinsics is not None:
|
| 165 |
+
_, _, h, w = image.shape
|
| 166 |
+
fallback_intrinsics = scale_intrinsics(
|
| 167 |
+
fx=moge2_intrinsics[0],
|
| 168 |
+
fy=moge2_intrinsics[1],
|
| 169 |
+
cx=moge2_intrinsics[2],
|
| 170 |
+
cy=moge2_intrinsics[3],
|
| 171 |
+
org_h=h,
|
| 172 |
+
org_w=w,
|
| 173 |
+
h=org_h,
|
| 174 |
+
w=org_w,
|
| 175 |
+
)
|
| 176 |
+
intrinsics_source = "MoGe-2 estimate"
|
| 177 |
+
else:
|
| 178 |
+
intrinsics_source = "default"
|
| 179 |
+
|
| 180 |
+
fx, fy, cx, cy = resolve_camera_intrinsics(
|
| 181 |
+
fx_org=fx_org,
|
| 182 |
+
fy_org=fy_org,
|
| 183 |
+
cx_org=cx_org,
|
| 184 |
+
cy_org=cy_org,
|
| 185 |
+
org_h=org_h,
|
| 186 |
+
org_w=org_w,
|
| 187 |
+
fallback_intrinsics=fallback_intrinsics,
|
| 188 |
+
)
|
| 189 |
+
return fx, fy, cx, cy, intrinsics_source
|
| 190 |
|
| 191 |
|
| 192 |
def build_scaled_intrinsics_matrix(
|
|
|
|
| 361 |
device=device,
|
| 362 |
).unsqueeze(0).expand(batch, -1, -1)
|
| 363 |
extrinsics = torch.eye(4, dtype=torch.float32, device=device).unsqueeze(0).expand(batch, -1, -1)
|
| 364 |
+
return fx, fy, cx, cy, intrinsics, extrinsics
|
InfiniDepth/utils/moge_utils.py
CHANGED
|
@@ -35,19 +35,40 @@ def _squeeze_hw(tensor: torch.Tensor, name: str) -> torch.Tensor:
|
|
| 35 |
raise ValueError(f"Unexpected {name} shape from MoGe-2: {tuple(tensor.shape)}")
|
| 36 |
|
| 37 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
@torch.no_grad()
|
| 39 |
-
def
|
| 40 |
image: torch.Tensor,
|
| 41 |
pretrained_model_name_or_path: str = "Ruicheng/moge-2-vitl-normal",
|
| 42 |
-
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 43 |
-
"""Run MoGe-2 and return dense pred depth
|
| 44 |
if image.ndim != 4 or image.shape[0] != 1 or image.shape[1] != 3:
|
| 45 |
raise ValueError(f"Expected image shape [1,3,H,W], got {tuple(image.shape)}")
|
| 46 |
|
| 47 |
device = image.device
|
| 48 |
model = _get_moge2_model(pretrained_model_name_or_path, device)
|
| 49 |
|
| 50 |
-
output = model.infer(image[0],apply_mask=True)
|
| 51 |
if "depth" not in output:
|
| 52 |
raise KeyError("MoGe-2 output missing key 'depth'.")
|
| 53 |
|
|
@@ -63,4 +84,42 @@ def estimate_metric_depth_with_moge2(
|
|
| 63 |
valid_mask = mask_hw & torch.isfinite(depth_hw) & (depth_hw > 0)
|
| 64 |
pred_depth = depth_hw * valid_mask.to(depth_hw.dtype)
|
| 65 |
|
| 66 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
raise ValueError(f"Unexpected {name} shape from MoGe-2: {tuple(tensor.shape)}")
|
| 36 |
|
| 37 |
|
| 38 |
+
def _squeeze_33(tensor: torch.Tensor, name: str) -> torch.Tensor:
|
| 39 |
+
if tensor.ndim == 2 and tensor.shape == (3, 3):
|
| 40 |
+
return tensor
|
| 41 |
+
if tensor.ndim == 3 and tensor.shape[0] == 1 and tensor.shape[1:] == (3, 3):
|
| 42 |
+
return tensor[0]
|
| 43 |
+
raise ValueError(f"Unexpected {name} shape from MoGe-2: {tuple(tensor.shape)}")
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _normalized_intrinsics_to_pixel_intrinsics(
|
| 47 |
+
intrinsics: torch.Tensor,
|
| 48 |
+
height: int,
|
| 49 |
+
width: int,
|
| 50 |
+
) -> tuple[float, float, float, float]:
|
| 51 |
+
intrinsics = _squeeze_33(intrinsics, "intrinsics")
|
| 52 |
+
fx = float(intrinsics[0, 0].item() * width)
|
| 53 |
+
fy = float(intrinsics[1, 1].item() * height)
|
| 54 |
+
cx = float(intrinsics[0, 2].item() * width - 0.5)
|
| 55 |
+
cy = float(intrinsics[1, 2].item() * height - 0.5)
|
| 56 |
+
return fx, fy, cx, cy
|
| 57 |
+
|
| 58 |
+
|
| 59 |
@torch.no_grad()
|
| 60 |
+
def estimate_metric_depth_and_intrinsics_with_moge2(
|
| 61 |
image: torch.Tensor,
|
| 62 |
pretrained_model_name_or_path: str = "Ruicheng/moge-2-vitl-normal",
|
| 63 |
+
) -> tuple[torch.Tensor, torch.Tensor, Optional[tuple[float, float, float, float]]]:
|
| 64 |
+
"""Run MoGe-2 and return dense pred depth, valid mask, and optional pixel intrinsics."""
|
| 65 |
if image.ndim != 4 or image.shape[0] != 1 or image.shape[1] != 3:
|
| 66 |
raise ValueError(f"Expected image shape [1,3,H,W], got {tuple(image.shape)}")
|
| 67 |
|
| 68 |
device = image.device
|
| 69 |
model = _get_moge2_model(pretrained_model_name_or_path, device)
|
| 70 |
|
| 71 |
+
output = model.infer(image[0], apply_mask=True)
|
| 72 |
if "depth" not in output:
|
| 73 |
raise KeyError("MoGe-2 output missing key 'depth'.")
|
| 74 |
|
|
|
|
| 84 |
valid_mask = mask_hw & torch.isfinite(depth_hw) & (depth_hw > 0)
|
| 85 |
pred_depth = depth_hw * valid_mask.to(depth_hw.dtype)
|
| 86 |
|
| 87 |
+
moge2_intrinsics = None
|
| 88 |
+
output_intrinsics = output.get("intrinsics")
|
| 89 |
+
if output_intrinsics is not None:
|
| 90 |
+
height, width = image.shape[-2:]
|
| 91 |
+
moge2_intrinsics = _normalized_intrinsics_to_pixel_intrinsics(
|
| 92 |
+
output_intrinsics.to(device=device, dtype=torch.float32),
|
| 93 |
+
height=height,
|
| 94 |
+
width=width,
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
return (
|
| 98 |
+
pred_depth.unsqueeze(0).unsqueeze(0),
|
| 99 |
+
valid_mask.to(torch.float32).unsqueeze(0).unsqueeze(0),
|
| 100 |
+
moge2_intrinsics,
|
| 101 |
+
)
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
@torch.no_grad()
|
| 105 |
+
def estimate_metric_depth_with_moge2(
|
| 106 |
+
image: torch.Tensor,
|
| 107 |
+
pretrained_model_name_or_path: str = "Ruicheng/moge-2-vitl-normal",
|
| 108 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 109 |
+
pred_depth, valid_mask, _ = estimate_metric_depth_and_intrinsics_with_moge2(
|
| 110 |
+
image=image,
|
| 111 |
+
pretrained_model_name_or_path=pretrained_model_name_or_path,
|
| 112 |
+
)
|
| 113 |
+
return pred_depth, valid_mask
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
@torch.no_grad()
|
| 117 |
+
def estimate_camera_intrinsics_with_moge2(
|
| 118 |
+
image: torch.Tensor,
|
| 119 |
+
pretrained_model_name_or_path: str = "Ruicheng/moge-2-vitl-normal",
|
| 120 |
+
) -> Optional[tuple[float, float, float, float]]:
|
| 121 |
+
_, _, intrinsics = estimate_metric_depth_and_intrinsics_with_moge2(
|
| 122 |
+
image=image,
|
| 123 |
+
pretrained_model_name_or_path=pretrained_model_name_or_path,
|
| 124 |
+
)
|
| 125 |
+
return intrinsics
|
app.py
CHANGED
|
@@ -23,7 +23,7 @@ import gradio as gr
|
|
| 23 |
import numpy as np
|
| 24 |
from PIL import Image
|
| 25 |
|
| 26 |
-
from InfiniDepth.utils.hf_demo_utils import ModelCache, prepare_runtime_assets,
|
| 27 |
from InfiniDepth.utils.logger import Log
|
| 28 |
|
| 29 |
try:
|
|
@@ -231,21 +231,22 @@ def _export_glb_from_points(xyz: np.ndarray, rgb: np.ndarray, output_path: Path)
|
|
| 231 |
cloud.export(output_path.as_posix())
|
| 232 |
|
| 233 |
|
| 234 |
-
|
|
|
|
| 235 |
image: np.ndarray,
|
| 236 |
depth_file,
|
| 237 |
model_type: str,
|
| 238 |
input_size: str,
|
| 239 |
output_resolution_mode: str,
|
| 240 |
upsample_ratio: int,
|
| 241 |
-
max_points_preview: int,
|
| 242 |
fx_org: Optional[float],
|
| 243 |
fy_org: Optional[float],
|
| 244 |
cx_org: Optional[float],
|
| 245 |
cy_org: Optional[float],
|
| 246 |
trace_path: str,
|
| 247 |
):
|
| 248 |
-
|
|
|
|
| 249 |
if image is None:
|
| 250 |
raise ValueError("Input RGB image is required")
|
| 251 |
|
|
@@ -253,15 +254,13 @@ def _run_demo_impl(
|
|
| 253 |
if depth_file is not None:
|
| 254 |
depth_path = depth_file if isinstance(depth_file, str) else depth_file.name
|
| 255 |
|
| 256 |
-
|
| 257 |
-
result = run_single_image_demo(
|
| 258 |
image_np=image,
|
| 259 |
depth_path=depth_path,
|
| 260 |
model_type=model_type,
|
| 261 |
input_size_text=input_size,
|
| 262 |
output_resolution_mode=output_resolution_mode,
|
| 263 |
upsample_ratio=int(upsample_ratio),
|
| 264 |
-
max_points_preview=int(max_points_preview),
|
| 265 |
fx_org=_none_if_invalid(fx_org),
|
| 266 |
fy_org=_none_if_invalid(fy_org),
|
| 267 |
cx_org=_none_if_invalid(cx_org),
|
|
@@ -269,68 +268,6 @@ def _run_demo_impl(
|
|
| 269 |
model_cache=MODEL_CACHE,
|
| 270 |
stage_callback=lambda stage: _append_trace(trace_path, stage),
|
| 271 |
)
|
| 272 |
-
_append_trace(trace_path, "worker:prepare_output_dir")
|
| 273 |
-
output_dir = _prepare_output_dir()
|
| 274 |
-
glb_path = output_dir / "pointcloud.glb"
|
| 275 |
-
ply_path = output_dir / "pointcloud.ply"
|
| 276 |
-
depth_vis_path = output_dir / "depth_colorized.png"
|
| 277 |
-
|
| 278 |
-
_append_trace(trace_path, "worker:export_glb")
|
| 279 |
-
_export_glb_from_points(result.xyz, result.rgb, glb_path)
|
| 280 |
-
if os.path.exists(result.ply_path):
|
| 281 |
-
shutil.copy2(result.ply_path, ply_path)
|
| 282 |
-
depth_vis_uint8 = result.depth_vis if result.depth_vis.dtype == np.uint8 else result.depth_vis.astype(np.uint8)
|
| 283 |
-
Image.fromarray(depth_vis_uint8).save(depth_vis_path)
|
| 284 |
-
|
| 285 |
-
depth_npy_path = output_dir / "depth.npy"
|
| 286 |
-
if os.path.exists(result.depth_npy_path):
|
| 287 |
-
shutil.copy2(result.depth_npy_path, depth_npy_path)
|
| 288 |
-
|
| 289 |
-
download_files = [glb_path.as_posix(), depth_vis_path.as_posix()]
|
| 290 |
-
if depth_npy_path.exists():
|
| 291 |
-
download_files.append(depth_npy_path.as_posix())
|
| 292 |
-
if ply_path.exists():
|
| 293 |
-
download_files.append(ply_path.as_posix())
|
| 294 |
-
|
| 295 |
-
status = (
|
| 296 |
-
f"Done. Output depth: {result.output_size[0]}x{result.output_size[1]}. "
|
| 297 |
-
f"Depth source: {result.depth_source_label}. "
|
| 298 |
-
f"Preview points: {result.xyz.shape[0]}. "
|
| 299 |
-
f"Camera intrinsics: {'default' if result.used_default_intrinsics else 'custom'}."
|
| 300 |
-
)
|
| 301 |
-
_append_trace(trace_path, "worker:completed")
|
| 302 |
-
return result.depth_vis, glb_path.as_posix(), download_files, status
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
@spaces.GPU(duration=180)
|
| 306 |
-
def run_demo_gpu(
|
| 307 |
-
image: np.ndarray,
|
| 308 |
-
depth_file,
|
| 309 |
-
model_type: str,
|
| 310 |
-
input_size: str,
|
| 311 |
-
output_resolution_mode: str,
|
| 312 |
-
upsample_ratio: int,
|
| 313 |
-
max_points_preview: int,
|
| 314 |
-
fx_org: Optional[float],
|
| 315 |
-
fy_org: Optional[float],
|
| 316 |
-
cx_org: Optional[float],
|
| 317 |
-
cy_org: Optional[float],
|
| 318 |
-
trace_path: str,
|
| 319 |
-
):
|
| 320 |
-
return _run_demo_impl(
|
| 321 |
-
image=image,
|
| 322 |
-
depth_file=depth_file,
|
| 323 |
-
model_type=model_type,
|
| 324 |
-
input_size=input_size,
|
| 325 |
-
output_resolution_mode=output_resolution_mode,
|
| 326 |
-
upsample_ratio=upsample_ratio,
|
| 327 |
-
max_points_preview=max_points_preview,
|
| 328 |
-
fx_org=fx_org,
|
| 329 |
-
fy_org=fy_org,
|
| 330 |
-
cx_org=cx_org,
|
| 331 |
-
cy_org=cy_org,
|
| 332 |
-
trace_path=trace_path,
|
| 333 |
-
)
|
| 334 |
|
| 335 |
|
| 336 |
def run_demo(
|
|
@@ -360,32 +297,88 @@ def run_demo(
|
|
| 360 |
f"depth_path={depth_path}, image_shape={image_shape}"
|
| 361 |
)
|
| 362 |
try:
|
| 363 |
-
|
|
|
|
| 364 |
image=image,
|
| 365 |
depth_file=depth_file,
|
| 366 |
model_type=model_type,
|
| 367 |
input_size=input_size,
|
| 368 |
output_resolution_mode=output_resolution_mode,
|
| 369 |
upsample_ratio=upsample_ratio,
|
| 370 |
-
max_points_preview=max_points_preview,
|
| 371 |
fx_org=fx_org,
|
| 372 |
fy_org=fy_org,
|
| 373 |
cx_org=cx_org,
|
| 374 |
cy_org=cy_org,
|
| 375 |
trace_path=trace_path,
|
| 376 |
)
|
| 377 |
-
|
| 378 |
-
|
| 379 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 380 |
except Exception as exc:
|
| 381 |
error_trace = traceback.format_exc()
|
| 382 |
trace_summary = _read_trace(trace_path)
|
| 383 |
Log.exception(f"[{request_id}] run_demo failed")
|
| 384 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 385 |
if trace_summary:
|
| 386 |
error_message = f"{error_message}\n\nLast worker stages:\n{trace_summary}"
|
| 387 |
if os.getenv("INFINIDEPTH_SHOW_TRACEBACK", "0") == "1":
|
| 388 |
error_message = f"{error_message}\n\n{error_trace}"
|
|
|
|
|
|
|
| 389 |
return None, None, [], error_message
|
| 390 |
|
| 391 |
|
|
@@ -449,6 +442,7 @@ with gr.Blocks(title="InfiniDepth Demo", theme=gr.themes.Soft(), css=CUSTOM_CSS,
|
|
| 449 |
run_button = gr.Button("Generate Depth + 3D", variant="primary", elem_id="run-btn")
|
| 450 |
gr.Markdown(
|
| 451 |
"Tips: when a depth map is uploaded it will be used automatically, otherwise the demo falls back to MoGe-2. "
|
|
|
|
| 452 |
"Use lower preview points for faster 3D interaction."
|
| 453 |
)
|
| 454 |
|
|
|
|
| 23 |
import numpy as np
|
| 24 |
from PIL import Image
|
| 25 |
|
| 26 |
+
from InfiniDepth.utils.hf_demo_utils import ModelCache, prepare_runtime_assets, run_gpu_inference, postprocess_gpu_result
|
| 27 |
from InfiniDepth.utils.logger import Log
|
| 28 |
|
| 29 |
try:
|
|
|
|
| 231 |
cloud.export(output_path.as_posix())
|
| 232 |
|
| 233 |
|
| 234 |
+
@spaces.GPU(duration=120)
|
| 235 |
+
def run_demo_gpu(
|
| 236 |
image: np.ndarray,
|
| 237 |
depth_file,
|
| 238 |
model_type: str,
|
| 239 |
input_size: str,
|
| 240 |
output_resolution_mode: str,
|
| 241 |
upsample_ratio: int,
|
|
|
|
| 242 |
fx_org: Optional[float],
|
| 243 |
fy_org: Optional[float],
|
| 244 |
cx_org: Optional[float],
|
| 245 |
cy_org: Optional[float],
|
| 246 |
trace_path: str,
|
| 247 |
):
|
| 248 |
+
"""GPU-only inference. Returns a GPUInferenceResult with all data on CPU."""
|
| 249 |
+
_append_trace(trace_path, "worker:entered run_demo_gpu")
|
| 250 |
if image is None:
|
| 251 |
raise ValueError("Input RGB image is required")
|
| 252 |
|
|
|
|
| 254 |
if depth_file is not None:
|
| 255 |
depth_path = depth_file if isinstance(depth_file, str) else depth_file.name
|
| 256 |
|
| 257 |
+
return run_gpu_inference(
|
|
|
|
| 258 |
image_np=image,
|
| 259 |
depth_path=depth_path,
|
| 260 |
model_type=model_type,
|
| 261 |
input_size_text=input_size,
|
| 262 |
output_resolution_mode=output_resolution_mode,
|
| 263 |
upsample_ratio=int(upsample_ratio),
|
|
|
|
| 264 |
fx_org=_none_if_invalid(fx_org),
|
| 265 |
fy_org=_none_if_invalid(fy_org),
|
| 266 |
cx_org=_none_if_invalid(cx_org),
|
|
|
|
| 268 |
model_cache=MODEL_CACHE,
|
| 269 |
stage_callback=lambda stage: _append_trace(trace_path, stage),
|
| 270 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 271 |
|
| 272 |
|
| 273 |
def run_demo(
|
|
|
|
| 297 |
f"depth_path={depth_path}, image_shape={image_shape}"
|
| 298 |
)
|
| 299 |
try:
|
| 300 |
+
# --- GPU-only inference (consumes ZeroGPU quota) ---
|
| 301 |
+
gpu_result = run_demo_gpu(
|
| 302 |
image=image,
|
| 303 |
depth_file=depth_file,
|
| 304 |
model_type=model_type,
|
| 305 |
input_size=input_size,
|
| 306 |
output_resolution_mode=output_resolution_mode,
|
| 307 |
upsample_ratio=upsample_ratio,
|
|
|
|
| 308 |
fx_org=fx_org,
|
| 309 |
fy_org=fy_org,
|
| 310 |
cx_org=cx_org,
|
| 311 |
cy_org=cy_org,
|
| 312 |
trace_path=trace_path,
|
| 313 |
)
|
| 314 |
+
_append_trace(trace_path, "ui:gpu_done, starting cpu postprocess")
|
| 315 |
+
|
| 316 |
+
# --- CPU post-processing (no GPU quota consumed) ---
|
| 317 |
+
result = postprocess_gpu_result(
|
| 318 |
+
gpu_result=gpu_result,
|
| 319 |
+
max_points_preview=int(max_points_preview),
|
| 320 |
+
stage_callback=lambda stage: _append_trace(trace_path, stage),
|
| 321 |
+
)
|
| 322 |
+
_append_trace(trace_path, "ui:postprocess_done, exporting files")
|
| 323 |
+
|
| 324 |
+
# --- File export ---
|
| 325 |
+
output_dir = _prepare_output_dir()
|
| 326 |
+
glb_path = output_dir / "pointcloud.glb"
|
| 327 |
+
ply_path = output_dir / "pointcloud.ply"
|
| 328 |
+
depth_vis_path = output_dir / "depth_colorized.png"
|
| 329 |
+
|
| 330 |
+
_export_glb_from_points(result.xyz, result.rgb, glb_path)
|
| 331 |
+
if os.path.exists(result.ply_path):
|
| 332 |
+
shutil.copy2(result.ply_path, ply_path)
|
| 333 |
+
depth_vis_uint8 = result.depth_vis if result.depth_vis.dtype == np.uint8 else result.depth_vis.astype(np.uint8)
|
| 334 |
+
Image.fromarray(depth_vis_uint8).save(depth_vis_path)
|
| 335 |
+
|
| 336 |
+
depth_npy_path = output_dir / "depth.npy"
|
| 337 |
+
if os.path.exists(result.depth_npy_path):
|
| 338 |
+
shutil.copy2(result.depth_npy_path, depth_npy_path)
|
| 339 |
+
|
| 340 |
+
download_files = [glb_path.as_posix(), depth_vis_path.as_posix()]
|
| 341 |
+
if depth_npy_path.exists():
|
| 342 |
+
download_files.append(depth_npy_path.as_posix())
|
| 343 |
+
if ply_path.exists():
|
| 344 |
+
download_files.append(ply_path.as_posix())
|
| 345 |
+
|
| 346 |
+
status = (
|
| 347 |
+
f"Done. Output depth: {result.output_size[0]}x{result.output_size[1]}. "
|
| 348 |
+
f"Depth source: {result.depth_source_label}. "
|
| 349 |
+
f"Preview points: {result.xyz.shape[0]}. "
|
| 350 |
+
f"Camera intrinsics: {result.intrinsics_source_label}."
|
| 351 |
+
)
|
| 352 |
+
_append_trace(trace_path, "ui:completed")
|
| 353 |
+
Log.info(f"[{request_id}] run_demo success: {status}")
|
| 354 |
+
return result.depth_vis, glb_path.as_posix(), download_files, status
|
| 355 |
except Exception as exc:
|
| 356 |
error_trace = traceback.format_exc()
|
| 357 |
trace_summary = _read_trace(trace_path)
|
| 358 |
Log.exception(f"[{request_id}] run_demo failed")
|
| 359 |
+
|
| 360 |
+
# Classify the error for clearer diagnostics
|
| 361 |
+
exc_type = type(exc).__name__
|
| 362 |
+
exc_module = type(exc).__module__ or ""
|
| 363 |
+
is_zerogpu_error = "spaces" in exc_module or "ZeroGPU" in str(exc) or "GPU task aborted" in str(exc)
|
| 364 |
+
if is_zerogpu_error:
|
| 365 |
+
error_message = (
|
| 366 |
+
f"[{request_id}] ZeroGPU error: {exc}\n\n"
|
| 367 |
+
"This is a HuggingFace ZeroGPU scheduling issue, not an inference bug.\n"
|
| 368 |
+
"Possible causes:\n"
|
| 369 |
+
" - GPU quota exhausted (wait for quota to reset)\n"
|
| 370 |
+
" - GPU task was preempted/aborted (try again)\n"
|
| 371 |
+
" - duration too high for remaining quota"
|
| 372 |
+
)
|
| 373 |
+
else:
|
| 374 |
+
error_message = f"Error [{request_id}] ({exc_type}): {exc}"
|
| 375 |
+
|
| 376 |
if trace_summary:
|
| 377 |
error_message = f"{error_message}\n\nLast worker stages:\n{trace_summary}"
|
| 378 |
if os.getenv("INFINIDEPTH_SHOW_TRACEBACK", "0") == "1":
|
| 379 |
error_message = f"{error_message}\n\n{error_trace}"
|
| 380 |
+
# Always log full traceback to server logs (visible in HF Space Logs tab)
|
| 381 |
+
Log.error(f"[{request_id}] Full traceback:\n{error_trace}")
|
| 382 |
return None, None, [], error_message
|
| 383 |
|
| 384 |
|
|
|
|
| 442 |
run_button = gr.Button("Generate Depth + 3D", variant="primary", elem_id="run-btn")
|
| 443 |
gr.Markdown(
|
| 444 |
"Tips: when a depth map is uploaded it will be used automatically, otherwise the demo falls back to MoGe-2. "
|
| 445 |
+
"If camera intrinsics are missing, the demo first tries MoGe-2 estimates before image-size defaults. "
|
| 446 |
"Use lower preview points for faster 3D interaction."
|
| 447 |
)
|
| 448 |
|