Upload depth_pro recipe (v1)

LICENSE

@@ -0,0 +1 @@
+The license of the original trained model can be found at https://huggingface.co/apple/DepthPro/blob/main/LICENSE.

README.md

@@ -0,0 +1,76 @@
+---
+library_name: pytorch
+license: other
+tags:
+- qai-hub-models
+- qualcomm
+- android
+pipeline_tag: depth-estimation
+---
+
+# DepthPro: Sharp monocular metric depth in less than a second, on-device
+
+Apple DepthPro is a zero-shot monocular metric depth estimator that emits high-resolution, sharp depth maps from a single 1536x1536 RGB image. The architecture is a multi-scale Vision Transformer built on Dinov2 encoders with DPT-style fusion; alongside the depth map the model predicts a per-image horizontal field of view, which downstream calibration converts into a focal length in pixels. This recipe wraps Apple's HuggingFace checkpoint (`apple/DepthPro-hf`, ~952M parameters) at the model's native 1536x1536 input resolution.
+
+This is based on the implementation of DepthPro found [here](https://github.com/apple/ml-depth-pro).
+This is a standalone recipe compatible with the [Qualcomm® AI Hub Models](https://github.com/quic/ai-hub-models) CLI — it can be compiled and evaluated on real Snapdragon devices via [Qualcomm® AI Hub Workbench](https://workbench.aihub.qualcomm.com).
+
+Qualcomm AI Hub Models uses [Qualcomm AI Hub Workbench](https://workbench.aihub.qualcomm.com) to compile, profile, and evaluate this model. [Sign up](https://myaccount.qualcomm.com/signup) to run these models on a hosted Qualcomm® device.
+
+
+## Setup
+### 1. Install the package
+Install the base package, fetch this recipe from Hugging Face, then use the
+`qai-hub-models` CLI to install the recipe's dependencies:
+```bash
+# NOTE: 3.10 <= PYTHON_VERSION < 3.14 is supported.
+pip install qai-hub-models
+qai-hub-models register ashwmurt/depth_pro
+qai-hub-models install depth_pro
+```
+`register` downloads the recipe and names it `depth_pro`, which is how every
+command below refers to it.
+
+### 2. Configure Qualcomm® AI Hub Workbench
+Sign-in to [Qualcomm® AI Hub Workbench](https://workbench.aihub.qualcomm.com/) with your
+Qualcomm® ID. Once signed in navigate to `Account -> Settings -> API Token`.
+
+With this API token, you can configure your client to run models on the cloud
+hosted devices.
+```bash
+qai-hub configure --api_token API_TOKEN
+```
+Navigate to [docs](https://workbench.aihub.qualcomm.com/docs/) for more information.
+
+## Run CLI Demo
+Run the following simple CLI demo to verify the model is working end to end:
+
+```bash
+qai-hub-models demo depth_pro
+```
+More details on the CLI tool can be found with the `--help` option. See
+[demo.py](demo.py) for sample usage of the model including pre/post processing
+scripts.
+
+By default, the demo will run locally in PyTorch. Pass `--eval-mode on-device` to run the model on a cloud-hosted target device.
+
+## Export for on-device deployment
+To run the model on Qualcomm® devices, you must export the model for use with an edge runtime such as
+TensorFlow Lite, ONNX Runtime, or Qualcomm AI Engine Direct.
+Use the following command to export the model:
+```bash
+qai-hub-models export depth_pro
+```
+Additional options are documented with the `--help` option.
+
+## License
+* The license for the original implementation of DepthPro can be found
+  [here](https://huggingface.co/apple/DepthPro/blob/main/LICENSE).
+
+## References
+* [Depth Pro: Sharp Monocular Metric Depth in Less Than a Second](https://arxiv.org/abs/2410.02073)
+* [Source Model Implementation](https://github.com/apple/ml-depth-pro)
+
+## Community
+* Join [our AI Hub Slack community](https://aihub.qualcomm.com/community/slack) to collaborate, post questions and learn more about on-device AI.
+* For questions or feedback please [reach out to us](mailto:ai-hub-support@qti.qualcomm.com).

__init__.py

@@ -0,0 +1,10 @@
+# ---------------------------------------------------------------------
+# Copyright (c) 2026 Qualcomm Technologies, Inc. and/or its subsidiaries.
+# SPDX-License-Identifier: BSD-3-Clause
+# ---------------------------------------------------------------------
+
+from .app import DepthProApp as App
+from .model import MODEL_ID
+from .model import DepthPro as Model
+
+__all__ = ["MODEL_ID", "App", "Model"]

app.py

@@ -0,0 +1,127 @@
+# ---------------------------------------------------------------------
+# Copyright (c) 2026 Qualcomm Technologies, Inc. and/or its subsidiaries.
+# SPDX-License-Identifier: BSD-3-Clause
+# ---------------------------------------------------------------------
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any, cast
+
+import matplotlib.pyplot as plt
+import numpy as np
+import numpy.typing as npt
+import torch
+from PIL import Image
+from torchvision import transforms
+
+from qai_hub_models.utils.image_processing import pil_resize_pad, undo_resize_pad
+
+
+@dataclass
+class DepthProPrediction:
+    """Structured output of :class:`DepthProApp`.
+
+    ``depth`` is metric depth in scene units (aligned via ``focal_length_px``);
+    ``heatmap`` is a plasma-colored visualization of inverse depth for display;
+    ``field_of_view`` is horizontal FoV in degrees; ``focal_length_px`` is
+    the pixel focal length derived from FoV and the original image width.
+    """
+
+    depth: npt.NDArray[np.float32]
+    heatmap: Image.Image
+    field_of_view: float
+    focal_length_px: float
+
+
+class DepthProApp:
+    """End-to-end app for Apple DepthPro depth estimation.
+
+    Wraps a callable returning ``(predicted_depth, field_of_view)`` — either
+    the torch model or an on-device runner. Preprocessing resizes with
+    aspect-preserving padding to the network's 1536x1536 input; post-
+    processing mirrors HuggingFace's
+    ``DepthProImageProcessorFast.post_process_depth_estimation`` (metric
+    scaling by ``width / focal_length_px``, then inversion of the canonical
+    inverse depth).
+    """
+
+    def __init__(
+        self,
+        model: Callable[
+            [torch.Tensor], tuple[torch.Tensor, torch.Tensor]
+        ],
+        input_height: int | None = None,
+        input_width: int | None = None,
+    ) -> None:
+        self.model = model
+        if input_height is None or input_width is None:
+            get_input_spec = getattr(model, "get_input_spec", None)
+            if get_input_spec is None:
+                raise TypeError(
+                    "DepthProApp needs input_height and input_width when the "
+                    "provided model is not a BaseModel (has no get_input_spec)."
+                )
+            _, _, h, w = get_input_spec()["image"][0]
+            input_height = input_height if input_height is not None else h
+            input_width = input_width if input_width is not None else w
+        self.input_height = input_height
+        self.input_width = input_width
+
+    def predict(self, *args: Any, **kwargs: Any) -> DepthProPrediction:
+        return self.estimate_depth(*args, **kwargs)
+
+    def estimate_depth(self, image: Image.Image) -> DepthProPrediction:
+        """Estimate depth, FoV, and focal length for a single image.
+
+        Parameters
+        ----------
+        image
+            PIL image in any resolution / aspect ratio.
+        """
+        resized_image, scale, padding = pil_resize_pad(
+            image, (self.input_height, self.input_width)
+        )
+        image_tensor = transforms.ToTensor()(resized_image).unsqueeze(0)
+        predicted_depth, field_of_view = self.model(image_tensor)
+
+        # Horizontal FoV -> focal length in pixels of the *original* image.
+        # Matches HF's post_process_depth_estimation.
+        orig_width = float(image.size[0])
+        fov_deg = field_of_view.detach().float().view(-1)
+        focal_length_px = 0.5 * orig_width / torch.tan(
+            0.5 * torch.deg2rad(fov_deg)
+        )
+
+        # Metric scaling of canonical inverse depth (again, from HF).
+        depth_scaled = predicted_depth * (
+            orig_width / focal_length_px
+        ).view(-1, 1, 1)
+
+        # (B, 1, H, W) shape is what undo_resize_pad expects.
+        depth_map = undo_resize_pad(
+            depth_scaled.unsqueeze(1), image.size, scale, padding
+        )
+
+        # Canonical inverse depth -> metric depth. Clamp mirrors HF.
+        depth_map = 1.0 / torch.clamp(depth_map, min=1e-4, max=1e4)
+
+        depth_np = cast(
+            npt.NDArray[np.float32],
+            depth_map.squeeze().detach().cpu().numpy().astype(np.float32),
+        )
+
+        # Visualize inverse depth so closer objects appear brighter, matching
+        # the shared depth-estimation demo convention.
+        inv = 1.0 / np.maximum(depth_np, 1e-6)
+        inv_norm = inv / max(inv.max(), 1e-6)
+        heatmap = plt.cm.get_cmap("plasma")(inv_norm)[..., :3]
+        heatmap_image = Image.fromarray((heatmap * 255).astype(np.uint8))
+
+        return DepthProPrediction(
+            depth=depth_np,
+            heatmap=heatmap_image,
+            field_of_view=float(fov_deg.squeeze().item()),
+            focal_length_px=float(focal_length_px.squeeze().item()),
+        )

demo.py

@@ -0,0 +1,60 @@
+# ---------------------------------------------------------------------
+# Copyright (c) 2026 Qualcomm Technologies, Inc. and/or its subsidiaries.
+# SPDX-License-Identifier: BSD-3-Clause
+# ---------------------------------------------------------------------
+
+from __future__ import annotations
+
+from qai_hub_models.utils.args import (
+    demo_model_from_cli_args,
+    get_model_cli_parser,
+    get_on_device_demo_parser,
+    validate_on_device_demo_args,
+)
+from qai_hub_models.utils.asset_loaders import CachedWebModelAsset, load_image
+from qai_hub_models.utils.display import display_or_save_image
+
+from .app import DepthProApp
+from .model import MODEL_ID, DepthPro
+
+# Reuse the midas depth-estimation fixture — any indoor/outdoor natural image
+# works; keeping this out-of-tree avoids uploading a fresh asset just for
+# the initial recipe.
+INPUT_IMAGE_ADDRESS = CachedWebModelAsset.from_asset_store(
+    "midas", 3, "test_input_image.jpg"
+)
+
+
+def main(is_test: bool = False) -> None:
+    parser = get_model_cli_parser(DepthPro)
+    parser = get_on_device_demo_parser(parser, add_output_dir=True)
+    parser.add_argument(
+        "--image",
+        type=str,
+        default=INPUT_IMAGE_ADDRESS,
+        help="image file path or URL",
+    )
+    args = parser.parse_args([] if is_test else None)
+    model = demo_model_from_cli_args(DepthPro, MODEL_ID, args)
+    validate_on_device_demo_args(args, MODEL_ID)
+
+    (_, _, height, width) = model.get_input_spec()["image"][0]
+    image = load_image(args.image)
+    print("Model Loaded")
+
+    app = DepthProApp(model, height, width)  # type: ignore[arg-type]
+    prediction = app.estimate_depth(image)
+
+    print(
+        f"Predicted field of view: {prediction.field_of_view:.2f} deg  "
+        f"(focal length: {prediction.focal_length_px:.1f} px)"
+    )
+
+    if not is_test:
+        display_or_save_image(
+            prediction.heatmap, args.output_dir, "out_heatmap.png", "heatmap"
+        )
+
+
+if __name__ == "__main__":
+    main()

manifest.yaml

@@ -0,0 +1,36 @@
+name: DepthPro
+id: depth_pro
+headline: Sharp monocular metric depth in less than a second, on-device.
+description: Apple DepthPro is a zero-shot monocular metric depth estimator that
+  emits high-resolution, sharp depth maps from a single 1536x1536 RGB image.
+  The architecture is a multi-scale Vision Transformer built on Dinov2 encoders
+  with DPT-style fusion; alongside the depth map the model predicts a per-image
+  horizontal field of view, which downstream calibration converts into a focal
+  length in pixels. This recipe wraps Apple's HuggingFace checkpoint
+  (`apple/DepthPro-hf`, ~952M parameters) at the model's native 1536x1536 input
+  resolution.
+domain: Computer Vision
+use_case: Depth Estimation
+applicable_scenarios:
+- Anomaly Detection
+- Inventory Management
+form_factors:
+- Phone
+- Tablet
+- IoT
+- XR
+has_static_banner: false
+has_animated_banner: false
+has_on_target_demo: true
+license_type: other-non-commercial
+license: https://huggingface.co/apple/DepthPro/blob/main/LICENSE
+source_repo: https://github.com/apple/ml-depth-pro
+research_paper: https://arxiv.org/abs/2410.02073
+research_paper_title: "Depth Pro: Sharp Monocular Metric Depth in Less Than a Second"
+related_models:
+- depth_anything_v2
+- midas
+templates:
+- depth_estimation
+supported_precisions:
+- float

model.py

@@ -0,0 +1,146 @@
+# ---------------------------------------------------------------------
+# Copyright (c) 2026 Qualcomm Technologies, Inc. and/or its subsidiaries.
+# SPDX-License-Identifier: BSD-3-Clause
+# ---------------------------------------------------------------------
+
+from __future__ import annotations
+
+import torch
+from transformers import DepthProForDepthEstimation
+from typing_extensions import Self
+
+from qai_hub_models.datasets.nyuv2 import NYUV2Dataset
+from qai_hub_models.models._shared.depth_estimation.depth_evaluator import (
+    DepthEvaluator,
+)
+from qai_hub_models.utils.base_dataset import BaseDataset
+from qai_hub_models.utils.base_evaluator import BaseEvaluator
+from qai_hub_models.utils.base_model import BaseModel
+from qai_hub_models.utils.input_spec import (
+    ColorFormat,
+    ImageMetadata,
+    InputSpec,
+    IoType,
+    OutputSpec,
+    TensorSpec,
+)
+
+MODEL_ID = "depth_pro"
+MODEL_ASSET_VERSION = 1
+DEFAULT_WEIGHTS = "apple/DepthPro-hf"
+DEFAULT_INPUT_SIZE = 1536
+
+
+class DepthProDepthEvaluator(DepthEvaluator):
+    """Adapts the shared depth δ1 evaluator to DepthPro's (depth, fov) tuple.
+
+    DepthPro's canonical inverse-depth head is what the shared evaluator
+    already scale/shift-aligns against NYUv2 ground truth, so the FoV output
+    is ignored here — focal-length calibration is only needed for absolute
+    metric alignment, which the δ1 metric is invariant to.
+    """
+
+    def add_batch(
+        self,
+        output: torch.Tensor | tuple[torch.Tensor, ...] | list[torch.Tensor],
+        gt: torch.Tensor,
+    ) -> None:
+        if isinstance(output, (tuple, list)):
+            output = output[0]
+        if output.dim() == 3:
+            # depth arrives as (B, H, W); base evaluator expects (B, 1, H, W)
+            output = output.unsqueeze(1)
+        super().add_batch(output, gt)
+
+
+class DepthPro(BaseModel):
+    """Apple DepthPro monocular metric depth estimator, end-to-end.
+
+    Exposes two on-device outputs: canonical inverse depth at the network's
+    input resolution and a scalar horizontal field of view (degrees) per
+    image. Off-device post-processing (see ``DepthProApp``) converts these
+    into metric depth and a focal length in pixels, matching HuggingFace's
+    ``DepthProImageProcessorFast.post_process_depth_estimation``.
+    """
+
+    def __init__(self, model: torch.nn.Module) -> None:
+        super().__init__()
+        self.model = model.eval()
+
+    @classmethod
+    def from_pretrained(cls, ckpt: str = DEFAULT_WEIGHTS) -> Self:
+        net = DepthProForDepthEstimation.from_pretrained(ckpt)
+        return cls(net)
+
+    def forward(
+        self, image: torch.Tensor
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        """Run DepthPro on `image`.
+
+        Parameters
+        ----------
+        image
+            Shape ``[B, 3, 1536, 1536]`` RGB in ``[0, 1]``.
+
+        Returns
+        -------
+        predicted_depth
+            Shape ``[B, 1536, 1536]``. Canonical inverse depth (the raw head
+            output before FoV-based metric scaling and inversion).
+        field_of_view
+            Shape ``[B]``, horizontal field of view in degrees.
+        """
+        # HF's DepthProImageProcessorFast rescales to [0, 1] then normalizes
+        # with mean=std=0.5, i.e. (image - 0.5) / 0.5 = 2 * image - 1.
+        pixel_values = image * 2.0 - 1.0
+        predicted_depth, field_of_view = self.model(
+            pixel_values, return_dict=False
+        )
+        return predicted_depth, field_of_view
+
+    def get_input_spec(
+        self,
+        batch_size: int = 1,
+        height: int = DEFAULT_INPUT_SIZE,
+        width: int = DEFAULT_INPUT_SIZE,
+    ) -> InputSpec:
+        return {
+            "image": TensorSpec(
+                shape=(batch_size, 3, height, width),
+                dtype="float32",
+                io_type=IoType.IMAGE,
+                value_range=(0.0, 1.0),
+                image_metadata=ImageMetadata(color_format=ColorFormat.RGB),
+                apply_runtime_channel_reordering=True,
+            ),
+        }
+
+    def get_output_spec(self) -> OutputSpec:
+        return {
+            "predicted_depth": TensorSpec(
+                io_type=IoType.TENSOR,
+                description=(
+                    "Canonical inverse depth at the network's input "
+                    "resolution. Invert and rescale by "
+                    "width / focal_length_px for metric depth."
+                ),
+                apply_runtime_channel_reordering=True,
+            ),
+            "field_of_view": TensorSpec(
+                io_type=IoType.TENSOR,
+                description=(
+                    "Horizontal field of view in degrees, one scalar per "
+                    "image; used off-device to derive focal length."
+                ),
+            ),
+        }
+
+    def get_evaluator(self) -> BaseEvaluator:
+        return DepthProDepthEvaluator()
+
+    @classmethod
+    def get_eval_dataset_classes(cls) -> list[type[BaseDataset]]:
+        return [NYUV2Dataset]
+
+    def get_calibration_dataset_cls(self) -> type[BaseDataset]:
+        return NYUV2Dataset

requirements.txt

@@ -0,0 +1 @@
+transformers==4.56.2

test.py

@@ -0,0 +1,44 @@
+# ---------------------------------------------------------------------
+# Copyright (c) 2026 Qualcomm Technologies, Inc. and/or its subsidiaries.
+# SPDX-License-Identifier: BSD-3-Clause
+# ---------------------------------------------------------------------
+
+from __future__ import annotations
+
+import numpy as np
+
+from qai_hub_models.utils.asset_loaders import load_image
+
+from .app import DepthProApp
+from .demo import INPUT_IMAGE_ADDRESS
+from .demo import main as demo_main
+from .model import DepthPro
+
+
+def test_task() -> None:
+    """Run torch DepthPro end-to-end on the sample fixture.
+
+    Sanity-checks that the pipeline resolves the HF weights, produces a
+    depth map at the network's native 1536x1536 unpadded to the original
+    input resolution, and yields a plausible field of view (roughly the
+    range Apple demos on natural imagery, 30-100 degrees).
+    """
+    model = DepthPro.from_pretrained()
+    (_, _, height, width) = model.get_input_spec()["image"][0]
+    app = DepthProApp(model, height, width)
+    image = load_image(INPUT_IMAGE_ADDRESS)
+    prediction = app.estimate_depth(image)
+
+    assert prediction.depth.ndim == 2
+    assert prediction.depth.shape == (image.size[1], image.size[0])
+    assert np.all(np.isfinite(prediction.depth))
+    assert prediction.depth.min() > 0
+
+    assert 10.0 < prediction.field_of_view < 170.0
+    assert prediction.focal_length_px > 0
+
+    assert prediction.heatmap.size == image.size
+
+
+def test_demo() -> None:
+    demo_main(is_test=True)