app.py

# ---------------------------------------------------------------------
# Copyright (c) 2026 Qualcomm Technologies, Inc. and/or its subsidiaries.
# SPDX-License-Identifier: BSD-3-Clause
# ---------------------------------------------------------------------

from __future__ import annotations

from collections.abc import Callable
from dataclasses import dataclass
from typing import Any, cast

import matplotlib.pyplot as plt
import numpy as np
import numpy.typing as npt
import torch
from PIL import Image
from torchvision import transforms

from qai_hub_models.utils.image_processing import pil_resize_pad, undo_resize_pad


@dataclass
class DepthProPrediction:
    """Structured output of :class:`DepthProApp`.

    ``depth`` is metric depth in scene units (aligned via ``focal_length_px``);
    ``heatmap`` is a plasma-colored visualization of inverse depth for display;
    ``field_of_view`` is horizontal FoV in degrees; ``focal_length_px`` is
    the pixel focal length derived from FoV and the original image width.
    """

    depth: npt.NDArray[np.float32]
    heatmap: Image.Image
    field_of_view: float
    focal_length_px: float


class DepthProApp:
    """End-to-end app for Apple DepthPro depth estimation.

    Wraps a callable returning ``(predicted_depth, field_of_view)`` — either
    the torch model or an on-device runner. Preprocessing resizes with
    aspect-preserving padding to the network's 1536x1536 input; post-
    processing mirrors HuggingFace's
    ``DepthProImageProcessorFast.post_process_depth_estimation`` (metric
    scaling by ``width / focal_length_px``, then inversion of the canonical
    inverse depth).
    """

    def __init__(
        self,
        model: Callable[
            [torch.Tensor], tuple[torch.Tensor, torch.Tensor]
        ],
        input_height: int | None = None,
        input_width: int | None = None,
    ) -> None:
        self.model = model
        if input_height is None or input_width is None:
            get_input_spec = getattr(model, "get_input_spec", None)
            if get_input_spec is None:
                raise TypeError(
                    "DepthProApp needs input_height and input_width when the "
                    "provided model is not a BaseModel (has no get_input_spec)."
                )
            _, _, h, w = get_input_spec()["image"][0]
            input_height = input_height if input_height is not None else h
            input_width = input_width if input_width is not None else w
        self.input_height = input_height
        self.input_width = input_width

    def predict(self, *args: Any, **kwargs: Any) -> DepthProPrediction:
        return self.estimate_depth(*args, **kwargs)

    def estimate_depth(self, image: Image.Image) -> DepthProPrediction:
        """Estimate depth, FoV, and focal length for a single image.

        Parameters
        ----------
        image
            PIL image in any resolution / aspect ratio.
        """
        resized_image, scale, padding = pil_resize_pad(
            image, (self.input_height, self.input_width)
        )
        image_tensor = transforms.ToTensor()(resized_image).unsqueeze(0)
        predicted_depth, field_of_view = self.model(image_tensor)

        # Horizontal FoV -> focal length in pixels of the *original* image.
        # Matches HF's post_process_depth_estimation.
        orig_width = float(image.size[0])
        fov_deg = field_of_view.detach().float().view(-1)
        focal_length_px = 0.5 * orig_width / torch.tan(
            0.5 * torch.deg2rad(fov_deg)
        )

        # Metric scaling of canonical inverse depth (again, from HF).
        depth_scaled = predicted_depth * (
            orig_width / focal_length_px
        ).view(-1, 1, 1)

        # (B, 1, H, W) shape is what undo_resize_pad expects.
        depth_map = undo_resize_pad(
            depth_scaled.unsqueeze(1), image.size, scale, padding
        )

        # Canonical inverse depth -> metric depth. Clamp mirrors HF.
        depth_map = 1.0 / torch.clamp(depth_map, min=1e-4, max=1e4)

        depth_np = cast(
            npt.NDArray[np.float32],
            depth_map.squeeze().detach().cpu().numpy().astype(np.float32),
        )

        # Visualize inverse depth so closer objects appear brighter, matching
        # the shared depth-estimation demo convention.
        inv = 1.0 / np.maximum(depth_np, 1e-6)
        inv_norm = inv / max(inv.max(), 1e-6)
        heatmap = plt.cm.get_cmap("plasma")(inv_norm)[..., :3]
        heatmap_image = Image.fromarray((heatmap * 255).astype(np.uint8))

        return DepthProPrediction(
            depth=depth_np,
            heatmap=heatmap_image,
            field_of_view=float(fov_deg.squeeze().item()),
            focal_length_px=float(focal_length_px.squeeze().item()),
        )