src/depth_anything_3/api.py

# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
# Modified 2026 by The PaGeR Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Depth Anything 3 API module.

This module provides the main API for Depth Anything 3, including model loading,
inference, and export capabilities. It supports both single and nested model architectures.
"""

from __future__ import annotations

import torch
import torch.nn as nn
from huggingface_hub import PyTorchModelHubMixin

from depth_anything_3.cfg import create_object, load_config
from depth_anything_3.registry import MODEL_REGISTRY
from depth_anything_3.utils.geometry import affine_inverse
from depth_anything_3.model.utils.valid_conv_padding import set_valid_pad_conv
torch.backends.cudnn.benchmark = False
# logger.info("CUDNN Benchmark Disabled")

SAFETENSORS_NAME = "model.safetensors"
CONFIG_NAME = "config.json"


class DepthAnything3(nn.Module, PyTorchModelHubMixin):
    """
    Depth Anything 3 main API class.

    This class provides a high-level interface for depth estimation using Depth Anything 3.
    It supports both single and nested model architectures with metric scaling capabilities.

    Features:
    - Hugging Face Hub integration via PyTorchModelHubMixin
    - Support for multiple model presets (vitb, vitg, nested variants)
    - Automatic mixed precision inference
    - Export capabilities for various formats (GLB, PLY, NPZ, etc.)
    - Camera pose estimation and metric depth scaling

    Usage:
        # Load from Hugging Face Hub
        model = DepthAnything3.from_pretrained("huggingface/model-name")

        # Or create with specific preset
        model = DepthAnything3(preset="vitg")

        # Run inference
        prediction = model.inference(images, export_dir="output", export_format="glb")
    """

    _commit_hash: str | None = None  # Set by mixin when loading from Hub

    def __init__(self, model_name: str = "da3-large", **kwargs):
        """
        Initialize DepthAnything3 with specified preset.

        Args:
        model_name: The name of the model preset to use.
                    Examples: 'da3-giant', 'da3-large', 'da3metric-large', 'da3nested-giant-large'.
        **kwargs: Additional keyword arguments (currently unused).
        """
        super().__init__()
        self.model_name = model_name

        # Build the underlying network from the bundled YAML preset.
        # Every released PaGeR checkpoint shares the same backbone topology
        # (DA3-Giant, no global PE / RoPE LoRA / seam-local-attn / fused-res
        # head ordering), so those flags are inlined and not re-read from
        # cfg.model. Only ``head_names``, ``valid_conv_padding`` and
        # ``log_depth`` differ between released variants.
        self.config = load_config(MODEL_REGISTRY[self.model_name])

        model_cfg = kwargs["model_cfg"]
        self.config.head["head_names"] = model_cfg.modalities
        self.config.head["valid_conv_padding"] = model_cfg.valid_conv_padding
        self.config.head["log_depth"] = model_cfg.log_depth
        self.config.head["with_confidence"] = True
        self.model = create_object(self.config)

        if model_cfg.valid_conv_padding:
            set_valid_pad_conv(self.model.head)

        # Device management (set by user)
        self.device = None

    def forward(
        self,
        image: torch.Tensor,
        extrinsics: torch.Tensor | None = None,
        intrinsics: torch.Tensor | None = None,
        face_ids: torch.Tensor | None = None,
        skip_heads=None,
    ) -> dict[str, torch.Tensor]:
        """
        Forward pass through the model.

        Args:
            image: Input batch with shape ``(B, N, 3, H, W)`` on the model device.
            extrinsics: Optional camera extrinsics with shape ``(B, N, 4, 4)``.
            intrinsics: Optional camera intrinsics with shape ``(B, N, 3, 3)``.
            face_ids: Optional (B, N) long tensor with values in [0, 5]; when
                the input only contains a subset of cubemap faces (N < 6),
                this tells the model which of the 6 canonical positions each
                N-slot occupies so the global PE / RoPE still line up.

        Returns:
            Dictionary containing model predictions
        """
        # Accept either (N, 4, 4) legacy input (broadcast across batch via
        # [None]) or (B, N, 4, 4) per-sample input. Same for intrinsics.
        ext_in = extrinsics if extrinsics.dim() == 4 else extrinsics[None]
        intr_in = intrinsics if intrinsics.dim() == 4 else intrinsics[None]
        ex_t_norm = self._normalize_extrinsics(ext_in.clone())
        prediction = self.model(
            image, ex_t_norm, intr_in, face_ids=face_ids, skip_heads=skip_heads,
        )
        return prediction
    

    def _normalize_extrinsics(self, ex_t: torch.Tensor | None) -> torch.Tensor | None:
        """Normalize extrinsics"""
        if ex_t is None:
            return None
        transform = affine_inverse(ex_t[:, :1])
        ex_t_norm = ex_t @ transform
        c2ws = affine_inverse(ex_t_norm)
        translations = c2ws[..., :3, 3]
        dists = translations.norm(dim=-1)
        median_dist = torch.median(dists)
        median_dist = torch.clamp(median_dist, min=1e-1)
        ex_t_norm[..., :3, 3] = ex_t_norm[..., :3, 3] / median_dist
        return ex_t_norm