model.py

# ---------------------------------------------------------------------
# Copyright (c) 2026 Qualcomm Technologies, Inc. and/or its subsidiaries.
# SPDX-License-Identifier: BSD-3-Clause
# ---------------------------------------------------------------------

from __future__ import annotations

import torch
from transformers import DepthProForDepthEstimation
from typing_extensions import Self

from qai_hub_models.datasets.nyuv2 import NYUV2Dataset
from qai_hub_models.models._shared.depth_estimation.depth_evaluator import (
    DepthEvaluator,
)
from qai_hub_models.utils.base_dataset import BaseDataset
from qai_hub_models.utils.base_evaluator import BaseEvaluator
from qai_hub_models.utils.base_model import BaseModel
from qai_hub_models.utils.input_spec import (
    ColorFormat,
    ImageMetadata,
    InputSpec,
    IoType,
    OutputSpec,
    TensorSpec,
)

MODEL_ID = "depth_pro"
MODEL_ASSET_VERSION = 1
DEFAULT_WEIGHTS = "apple/DepthPro-hf"
DEFAULT_INPUT_SIZE = 1536


class DepthProDepthEvaluator(DepthEvaluator):
    """Adapts the shared depth δ1 evaluator to DepthPro's (depth, fov) tuple.

    DepthPro's canonical inverse-depth head is what the shared evaluator
    already scale/shift-aligns against NYUv2 ground truth, so the FoV output
    is ignored here — focal-length calibration is only needed for absolute
    metric alignment, which the δ1 metric is invariant to.
    """

    def add_batch(
        self,
        output: torch.Tensor | tuple[torch.Tensor, ...] | list[torch.Tensor],
        gt: torch.Tensor,
    ) -> None:
        if isinstance(output, (tuple, list)):
            output = output[0]
        if output.dim() == 3:
            # depth arrives as (B, H, W); base evaluator expects (B, 1, H, W)
            output = output.unsqueeze(1)
        super().add_batch(output, gt)


class DepthPro(BaseModel):
    """Apple DepthPro monocular metric depth estimator, end-to-end.

    Exposes two on-device outputs: canonical inverse depth at the network's
    input resolution and a scalar horizontal field of view (degrees) per
    image. Off-device post-processing (see ``DepthProApp``) converts these
    into metric depth and a focal length in pixels, matching HuggingFace's
    ``DepthProImageProcessorFast.post_process_depth_estimation``.
    """

    def __init__(self, model: torch.nn.Module) -> None:
        super().__init__()
        self.model = model.eval()

    @classmethod
    def from_pretrained(cls, ckpt: str = DEFAULT_WEIGHTS) -> Self:
        net = DepthProForDepthEstimation.from_pretrained(ckpt)
        return cls(net)

    def forward(
        self, image: torch.Tensor
    ) -> tuple[torch.Tensor, torch.Tensor]:
        """Run DepthPro on `image`.

        Parameters
        ----------
        image
            Shape ``[B, 3, 1536, 1536]`` RGB in ``[0, 1]``.

        Returns
        -------
        predicted_depth
            Shape ``[B, 1536, 1536]``. Canonical inverse depth (the raw head
            output before FoV-based metric scaling and inversion).
        field_of_view
            Shape ``[B]``, horizontal field of view in degrees.
        """
        # HF's DepthProImageProcessorFast rescales to [0, 1] then normalizes
        # with mean=std=0.5, i.e. (image - 0.5) / 0.5 = 2 * image - 1.
        pixel_values = image * 2.0 - 1.0
        predicted_depth, field_of_view = self.model(
            pixel_values, return_dict=False
        )
        return predicted_depth, field_of_view

    def get_input_spec(
        self,
        batch_size: int = 1,
        height: int = DEFAULT_INPUT_SIZE,
        width: int = DEFAULT_INPUT_SIZE,
    ) -> InputSpec:
        return {
            "image": TensorSpec(
                shape=(batch_size, 3, height, width),
                dtype="float32",
                io_type=IoType.IMAGE,
                value_range=(0.0, 1.0),
                image_metadata=ImageMetadata(color_format=ColorFormat.RGB),
                apply_runtime_channel_reordering=True,
            ),
        }

    def get_output_spec(self) -> OutputSpec:
        return {
            "predicted_depth": TensorSpec(
                io_type=IoType.TENSOR,
                description=(
                    "Canonical inverse depth at the network's input "
                    "resolution. Invert and rescale by "
                    "width / focal_length_px for metric depth."
                ),
                apply_runtime_channel_reordering=True,
            ),
            "field_of_view": TensorSpec(
                io_type=IoType.TENSOR,
                description=(
                    "Horizontal field of view in degrees, one scalar per "
                    "image; used off-device to derive focal length."
                ),
            ),
        }

    def get_evaluator(self) -> BaseEvaluator:
        return DepthProDepthEvaluator()

    @classmethod
    def get_eval_dataset_classes(cls) -> list[type[BaseDataset]]:
        return [NYUV2Dataset]

    def get_calibration_dataset_cls(self) -> type[BaseDataset]:
        return NYUV2Dataset