Initial MetaView novel view synthesis demo

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/1.png filter=lfs diff=lfs merge=lfs -text
+examples/12.png filter=lfs diff=lfs merge=lfs -text
+examples/5.png filter=lfs diff=lfs merge=lfs -text
+examples/9.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,13 +1,32 @@
 ---
-title: Metaview
-emoji: 🐢
-colorFrom: gray
-colorTo: green
+title: MetaView Novel View Synthesis
+emoji: 🎥
+colorFrom: pink
+colorTo: purple
 sdk: gradio
-sdk_version: 6.20.0
-python_version: '3.12'
+sdk_version: 5.49.1
 app_file: app.py
 pinned: false
+license: apache-2.0
+short_description: Monocular novel view synthesis from a single image
+python_version: "3.10"
+startup_duration_timeout: 1h
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# MetaView — Monocular Novel View Synthesis
+
+Interactive demo for [**MetaView**](https://huggingface.co/Kwai-Kolors/MetaView)
+(ECCV 2026): *Monocular Novel View Synthesis with Scale-Aware Implicit Geometry
+Priors*.
+
+Upload a single image, choose a target camera **yaw** (left/right) and **pitch**
+(up/down), and MetaView renders the scene from the new viewpoint.
+
+The pipeline combines:
+- **Qwen-Image-Edit** MM-DiT backbone (novel-view generation),
+- **Depth-Anything-3** GIANT (implicit 3D feature priors) + NESTED (dense depth),
+- **Modified RoPE (PRoPE)** for metric scale anchoring of the camera pose.
+
+Links: [Model](https://huggingface.co/Kwai-Kolors/MetaView) ·
+[Code](https://github.com/KlingAIResearch/MetaView) ·
+[Paper](https://arxiv.org/abs/2607.12000)

app.py

@@ -0,0 +1,270 @@
+import os
+
+# Allocator config to survive transient memory spikes from the large DiT.
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+# DiffSynth defaults to ModelScope; force Hugging Face so all weights come from the Hub.
+os.environ.setdefault("DIFFSYNTH_DOWNLOAD_SOURCE", "huggingface")
+
+import glob
+import math
+import sys
+import tempfile
+
+import spaces  # must come before torch / any CUDA-touching import
+import numpy as np
+import torch
+import torch.nn.functional as F
+import gradio as gr
+from PIL import Image
+from huggingface_hub import snapshot_download
+
+# Local vendored packages (MetaView `src/` + `diffsynth/` and Depth-Anything-3).
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from depth_anything_3.api import DepthAnything3
+from diffsynth.core import ModelConfig
+from diffsynth import load_state_dict
+from src.MetaView_pipeline import MetaViewPipeline
+
+# ---------------------------------------------------------------------------
+# Model ids
+# ---------------------------------------------------------------------------
+METAVIEW_REPO = "Kwai-Kolors/MetaView"
+DA3_GIANT_REPO = "depth-anything/DA3-GIANT-1.1"
+DA3_NESTED_REPO = "depth-anything/DA3NESTED-GIANT-LARGE-1.1"
+QWEN_EDIT_REPO = "Qwen/Qwen-Image-Edit"
+
+# MetaView global config (matches the reference inference script).
+EXPORT_3D_FEAT_LAYERS = [19, 27, 33, 39]
+PROPE_DIM_ARRANGE = [64, 20, 20, 24]
+ADD_DEPTH = len(PROPE_DIM_ARRANGE) == 4
+MERGE_3D = True
+PROMPT = ["镜头视角转到指定位置"]  # "move the camera view to the target position"
+GEN_W, GEN_H = 960, 528
+NUM_STEPS = 40
+
+device = "cuda"
+dtype = torch.bfloat16
+
+# ---------------------------------------------------------------------------
+# Load everything at module scope. `import spaces` above has hijacked
+# torch.cuda.*, so `.to("cuda")` here packs weights to disk and streams them
+# into VRAM on the first @spaces.GPU call.
+# ---------------------------------------------------------------------------
+print("[*] Downloading MetaView checkpoint...")
+metaview_dir = snapshot_download(METAVIEW_REPO)
+metaview_ckpt = glob.glob(os.path.join(metaview_dir, "*.safetensors"))[0]
+
+print("[*] Downloading Depth-Anything-3 models...")
+da3_giant_dir = snapshot_download(DA3_GIANT_REPO)
+da3_nested_dir = snapshot_download(DA3_NESTED_REPO)
+
+# Download the Qwen-Image-Edit backbone components from the Hub explicitly so we
+# can hand DiffSynth concrete local file paths (avoids the ModelScope default
+# download path and the processor-dir globbing quirk).
+print("[*] Downloading Qwen-Image-Edit backbone...")
+qwen_edit_dir = snapshot_download(
+    QWEN_EDIT_REPO,
+    allow_patterns=["transformer/diffusion_pytorch_model*.safetensors",
+                    "transformer/*.json", "processor/*"],
+)
+qwen_image_dir = snapshot_download(
+    "Qwen/Qwen-Image",
+    allow_patterns=["text_encoder/model*.safetensors", "text_encoder/*.json",
+                    "vae/diffusion_pytorch_model.safetensors", "vae/*.json",
+                    "tokenizer/*"],
+)
+
+transformer_files = sorted(glob.glob(
+    os.path.join(qwen_edit_dir, "transformer", "diffusion_pytorch_model*.safetensors")))
+text_encoder_files = sorted(glob.glob(
+    os.path.join(qwen_image_dir, "text_encoder", "model*.safetensors")))
+vae_file = os.path.join(qwen_image_dir, "vae", "diffusion_pytorch_model.safetensors")
+processor_path = os.path.join(qwen_edit_dir, "processor")
+tokenizer_path = os.path.join(qwen_image_dir, "tokenizer")
+
+print("[*] Loading Depth-Anything-3 GIANT (3D feature extractor)...")
+da3_giant = DepthAnything3.from_pretrained(da3_giant_dir).to(device=device).eval()
+
+print("[*] Loading Depth-Anything-3 NESTED (dense depth)...")
+da3_nested = DepthAnything3.from_pretrained(da3_nested_dir).to(device=device).eval()
+
+print("[*] Loading MetaView / Qwen-Image-Edit pipeline...")
+pipe = MetaViewPipeline.from_pretrained(
+    torch_dtype=dtype,
+    device=device,
+    model_configs=[
+        ModelConfig(path=transformer_files),
+        ModelConfig(path=text_encoder_files),
+        ModelConfig(path=vae_file),
+    ],
+    tokenizer_config=ModelConfig(path=tokenizer_path),
+    processor_config=ModelConfig(path=processor_path),
+)
+
+print(f"[*] Applying MetaView weights from {metaview_ckpt}...")
+state_dict = load_state_dict(metaview_ckpt)
+pipe.dit.load_state_dict(state_dict, strict=False)
+del state_dict
+print("[*] All models loaded.")
+
+
+def compute_target_extrinsic(yaw_deg, pitch_deg, radius):
+    """Camera World-to-Camera extrinsic for a rotation around a sphere center
+    in front of the camera (yaw = left/right, pitch = up/down)."""
+    yaw = math.radians(yaw_deg)
+    pitch = math.radians(pitch_deg)
+    R_y = np.array([[np.cos(yaw), 0, np.sin(yaw)],
+                    [0, 1, 0],
+                    [-np.sin(yaw), 0, np.cos(yaw)]])
+    R_x = np.array([[1, 0, 0],
+                    [0, np.cos(pitch), -np.sin(pitch)],
+                    [0, np.sin(pitch), np.cos(pitch)]])
+    R = R_y @ R_x
+    C = np.array([0.0, 0.0, radius])
+    t = C - R @ C
+    T = np.eye(4)
+    T[:3, :3] = R
+    T[:3, 3] = t
+    return T
+
+
+@spaces.GPU(duration=150, size="xlarge")
+def synthesize(image, yaw, pitch, radius, progress=gr.Progress(track_tqdm=True)):
+    """Synthesize a novel view of a single input image at a target camera pose.
+
+    Args:
+        image: Source image (a single monocular view).
+        yaw: Horizontal camera rotation in degrees (positive = right, negative = left).
+        pitch: Vertical camera rotation in degrees (positive = up, negative = down).
+        radius: Rotation radius. If 0, it is auto-derived from the scene center depth.
+
+    Returns:
+        The synthesized novel-view image at the requested camera pose.
+    """
+    if image is None:
+        raise gr.Error("Please provide an input image.")
+
+    original_image = image.convert("RGB")
+    edit_image = original_image.resize((GEN_W, GEN_H))
+
+    with torch.inference_mode():
+        # --- 1. 3D feature extraction (DA3 GIANT) + intrinsics ---
+        feat_out = da3_giant.inference(
+            [edit_image], export_feat_layers=EXPORT_3D_FEAT_LAYERS, process_res=840
+        )
+        intri = feat_out.intrinsics[0]
+        width = intri[0, 2] * 2
+        height = intri[1, 2] * 2
+        Ks_matrix = [
+            [intri[0, 0] / width, 0.0, 0.0],
+            [0.0, intri[1, 1] / height, 0.0],
+            [0.0, 0.0, 1.0],
+        ]
+        Ks = torch.Tensor(Ks_matrix)
+        Ks = torch.stack([Ks, Ks], dim=0).unsqueeze(0)  # (1, 2, 3, 3)
+
+        feats = [torch.from_numpy(feat_out.aux[f"feat_layer_{layer}"])
+                 for layer in EXPORT_3D_FEAT_LAYERS]
+        feat_3D = torch.cat(feats, dim=-1).to(dtype=dtype, device=device)
+
+        # --- 2. Dense depth estimation (DA3 NESTED) ---
+        prediction = da3_nested.inference([edit_image], process_res=840)
+        depth_edit = torch.Tensor(prediction.depth).unsqueeze(0)
+        depth_edit = F.interpolate(depth_edit, size=(GEN_H, GEN_W),
+                                   mode="bilinear", align_corners=False)[0]
+        depth_latent = torch.zeros_like(depth_edit)
+        depth = torch.cat([depth_latent, depth_edit], dim=0).unsqueeze(0)  # (1, 2, H, W)
+
+        # --- 3. Target pose ---
+        r = float(radius)
+        if r <= 0:
+            depth_squeeze = depth[0, 1]
+            r = depth_squeeze[depth_squeeze.shape[0] // 2,
+                              depth_squeeze.shape[1] // 2].item()
+        extrinsic_target = compute_target_extrinsic(float(yaw), float(pitch), r)
+        extrinsic_source = np.eye(4)
+        viewmats = torch.Tensor(
+            np.stack((extrinsic_target, extrinsic_source), axis=0)
+        ).unsqueeze(0)  # (1, 2, 4, 4) -> [target, source]
+
+        # --- 4. Novel view generation (MetaView DiT) ---
+        generated_image = pipe(
+            PROMPT, edit_image=edit_image, edit_image_auto_resize=False,
+            seed=0,
+            viewmats=viewmats.to(device=device, dtype=dtype),
+            Ks=Ks.to(device=device, dtype=dtype),
+            prope_dim_arrange=PROPE_DIM_ARRANGE,
+            add_attn=True,
+            add_3D=True,
+            feat_3D=feat_3D,
+            depth=depth.to(device=device, dtype=dtype) if ADD_DEPTH else None,
+            merge_3D=MERGE_3D,
+            val=True,
+            num_inference_steps=NUM_STEPS,
+            height=GEN_H, width=GEN_W,
+        )
+
+    return generated_image
+
+
+CSS = """
+#col-container { max-width: 1100px; margin: 0 auto; }
+.dark .gradio-container { color: var(--body-text-color); }
+"""
+
+with gr.Blocks(theme=gr.themes.Citrus(), css=CSS) as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.Markdown(
+            """
+            # MetaView — Monocular Novel View Synthesis
+            Synthesize a **novel camera view** from a single image. Upload an image,
+            pick a target camera **yaw** (left/right) and **pitch** (up/down), and
+            MetaView renders the scene from that new viewpoint.
+
+            Built on Qwen-Image-Edit + Depth-Anything-3 geometry priors.
+            [Model](https://huggingface.co/Kwai-Kolors/MetaView) ·
+            [Code](https://github.com/KlingAIResearch/MetaView) ·
+            [Paper](https://arxiv.org/abs/2607.12000)
+            """
+        )
+        with gr.Row():
+            with gr.Column():
+                image = gr.Image(label="Input image", type="pil", height=340)
+                with gr.Row():
+                    yaw = gr.Slider(-60, 60, value=-30, step=1,
+                                    label="Yaw (°)  ←  left | right  →")
+                    pitch = gr.Slider(-45, 45, value=10, step=1,
+                                      label="Pitch (°)  ↓ down | up ↑")
+                run = gr.Button("Synthesize novel view", variant="primary")
+                with gr.Accordion("Advanced settings", open=False):
+                    radius = gr.Slider(
+                        0.0, 10.0, value=0.0, step=0.1,
+                        label="Rotation radius (0 = auto from center depth)",
+                    )
+            with gr.Column():
+                output = gr.Image(label="Novel view", height=340)
+
+        gr.Examples(
+            examples=[
+                ["examples/1.png", -30, 10],
+                ["examples/5.png", 30, 0],
+                ["examples/9.png", -25, 15],
+                ["examples/12.png", 40, -10],
+            ],
+            inputs=[image, yaw, pitch],
+            outputs=output,
+            fn=synthesize,
+            cache_examples=True,
+            cache_mode="lazy",
+        )
+
+    run.click(
+        synthesize,
+        inputs=[image, yaw, pitch, radius],
+        outputs=output,
+        api_name="synthesize",
+    )
+
+if __name__ == "__main__":
+    demo.launch(mcp_server=True)

depth_anything_3/api.py

@@ -0,0 +1,451 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Depth Anything 3 API module.
+
+This module provides the main API for Depth Anything 3, including model loading,
+inference, and export capabilities. It supports both single and nested model architectures.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Optional, Sequence
+import numpy as np
+import torch
+import torch.nn as nn
+from huggingface_hub import PyTorchModelHubMixin
+from PIL import Image
+
+from depth_anything_3.cfg import create_object, load_config
+from depth_anything_3.registry import MODEL_REGISTRY
+from depth_anything_3.specs import Prediction
+from depth_anything_3.utils.geometry import affine_inverse
+from depth_anything_3.utils.io.input_processor import InputProcessor
+from depth_anything_3.utils.io.output_processor import OutputProcessor
+from depth_anything_3.utils.logger import logger
+
+# NOTE: `export` (gsplat / moviepy / open3d / pycolmap) and `align_poses_umeyama`
+# (evo) are only used in the export / pose-alignment code paths that this demo
+# never exercises. Import them lazily inside the functions that use them so the
+# heavy, hard-to-build deps aren't required at module import time.
+
+torch.backends.cudnn.benchmark = False
+# logger.info("CUDNN Benchmark Disabled")
+
+SAFETENSORS_NAME = "model.safetensors"
+CONFIG_NAME = "config.json"
+
+
+class DepthAnything3(nn.Module, PyTorchModelHubMixin):
+    """
+    Depth Anything 3 main API class.
+
+    This class provides a high-level interface for depth estimation using Depth Anything 3.
+    It supports both single and nested model architectures with metric scaling capabilities.
+
+    Features:
+    - Hugging Face Hub integration via PyTorchModelHubMixin
+    - Support for multiple model presets (vitb, vitg, nested variants)
+    - Automatic mixed precision inference
+    - Export capabilities for various formats (GLB, PLY, NPZ, etc.)
+    - Camera pose estimation and metric depth scaling
+
+    Usage:
+        # Load from Hugging Face Hub
+        model = DepthAnything3.from_pretrained("huggingface/model-name")
+
+        # Or create with specific preset
+        model = DepthAnything3(preset="vitg")
+
+        # Run inference
+        prediction = model.inference(images, export_dir="output", export_format="glb")
+    """
+
+    _commit_hash: str | None = None  # Set by mixin when loading from Hub
+
+    def __init__(self, model_name: str = "da3-large", **kwargs):
+        """
+        Initialize DepthAnything3 with specified preset.
+
+        Args:
+        model_name: The name of the model preset to use.
+                    Examples: 'da3-giant', 'da3-large', 'da3metric-large', 'da3nested-giant-large'.
+        **kwargs: Additional keyword arguments (currently unused).
+        """
+        super().__init__()
+        self.model_name = model_name
+
+        # Build the underlying network
+        self.config = load_config(MODEL_REGISTRY[self.model_name])
+        self.model = create_object(self.config)
+        self.model.eval()
+
+        # Initialize processors
+        self.input_processor = InputProcessor()
+        self.output_processor = OutputProcessor()
+
+        # Device management (set by user)
+        self.device = None
+
+    @torch.inference_mode()
+    def forward(
+        self,
+        image: torch.Tensor,
+        extrinsics: torch.Tensor | None = None,
+        intrinsics: torch.Tensor | None = None,
+        export_feat_layers: list[int] | None = None,
+        infer_gs: bool = False,
+        use_ray_pose: bool = False,
+        ref_view_strategy: str = "saddle_balanced",
+    ) -> dict[str, torch.Tensor]:
+        """
+        Forward pass through the model.
+
+        Args:
+            image: Input batch with shape ``(B, N, 3, H, W)`` on the model device.
+            extrinsics: Optional camera extrinsics with shape ``(B, N, 4, 4)``.
+            intrinsics: Optional camera intrinsics with shape ``(B, N, 3, 3)``.
+            export_feat_layers: Layer indices to return intermediate features for.
+            infer_gs: Enable Gaussian Splatting branch.
+            use_ray_pose: Use ray-based pose estimation instead of camera decoder.
+            ref_view_strategy: Strategy for selecting reference view from multiple views.
+
+        Returns:
+            Dictionary containing model predictions
+        """
+        # Determine optimal autocast dtype
+        autocast_dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
+        with torch.no_grad():
+            with torch.autocast(device_type=image.device.type, dtype=autocast_dtype):
+                return self.model(
+                    image, extrinsics, intrinsics, export_feat_layers, infer_gs, use_ray_pose, ref_view_strategy
+                )
+
+    def inference(
+        self,
+        image: list[np.ndarray | Image.Image | str],
+        extrinsics: np.ndarray | None = None,
+        intrinsics: np.ndarray | None = None,
+        align_to_input_ext_scale: bool = True,
+        infer_gs: bool = False,
+        use_ray_pose: bool = False,
+        ref_view_strategy: str = "saddle_balanced",
+        render_exts: np.ndarray | None = None,
+        render_ixts: np.ndarray | None = None,
+        render_hw: tuple[int, int] | None = None,
+        process_res: int = 504,
+        process_res_method: str = "upper_bound_resize",
+        export_dir: str | None = None,
+        export_format: str = "mini_npz",
+        export_feat_layers: Sequence[int] | None = None,
+        # GLB export parameters
+        conf_thresh_percentile: float = 40.0,
+        num_max_points: int = 1_000_000,
+        show_cameras: bool = True,
+        # Feat_vis export parameters
+        feat_vis_fps: int = 15,
+        # Other export parameters, e.g., gs_ply, gs_video
+        export_kwargs: Optional[dict] = {},
+    ) -> Prediction:
+        """
+        Run inference on input images.
+
+        Args:
+            image: List of input images (numpy arrays, PIL Images, or file paths)
+            extrinsics: Camera extrinsics (N, 4, 4)
+            intrinsics: Camera intrinsics (N, 3, 3)
+            align_to_input_ext_scale: whether to align the input pose scale to the prediction
+            infer_gs: Enable the 3D Gaussian branch (needed for `gs_ply`/`gs_video` exports)
+            use_ray_pose: Use ray-based pose estimation instead of camera decoder (default: False)
+            ref_view_strategy: Strategy for selecting reference view from multiple views.
+                Options: "first", "middle", "saddle_balanced", "saddle_sim_range".
+                Default: "saddle_balanced". For single view input (S ≤ 2), no reordering is performed.
+            render_exts: Optional render extrinsics for Gaussian video export
+            render_ixts: Optional render intrinsics for Gaussian video export
+            render_hw: Optional render resolution for Gaussian video export
+            process_res: Processing resolution
+            process_res_method: Resize method for processing
+            export_dir: Directory to export results
+            export_format: Export format (mini_npz, npz, glb, ply, gs, gs_video)
+            export_feat_layers: Layer indices to export intermediate features from
+            conf_thresh_percentile: [GLB] Lower percentile for adaptive confidence threshold (default: 40.0) # noqa: E501
+            num_max_points: [GLB] Maximum number of points in the point cloud (default: 1,000,000)
+            show_cameras: [GLB] Show camera wireframes in the exported scene (default: True)
+            feat_vis_fps: [FEAT_VIS] Frame rate for output video (default: 15)
+            export_kwargs: additional arguments to export functions.
+
+        Returns:
+            Prediction object containing depth maps and camera parameters
+        """
+        if "gs" in export_format:
+            assert infer_gs, "must set `infer_gs=True` to perform gs-related export."
+
+        if "colmap" in export_format:
+            assert isinstance(image[0], str), "`image` must be image paths for COLMAP export."
+
+        # Preprocess images
+        imgs_cpu, extrinsics, intrinsics = self._preprocess_inputs(
+            image, extrinsics, intrinsics, process_res, process_res_method
+        )
+
+        # Prepare tensors for model
+        imgs, ex_t, in_t = self._prepare_model_inputs(imgs_cpu, extrinsics, intrinsics)
+
+        # Normalize extrinsics
+        ex_t_norm = self._normalize_extrinsics(ex_t.clone() if ex_t is not None else None)
+
+        # Run model forward pass
+        export_feat_layers = list(export_feat_layers) if export_feat_layers is not None else []
+
+        raw_output = self._run_model_forward(
+            imgs, ex_t_norm, in_t, export_feat_layers, infer_gs, use_ray_pose, ref_view_strategy
+        )
+
+        # Convert raw output to prediction
+        prediction = self._convert_to_prediction(raw_output)
+
+        # Align prediction to extrinsincs
+        prediction = self._align_to_input_extrinsics_intrinsics(
+            extrinsics, intrinsics, prediction, align_to_input_ext_scale
+        )
+
+        # Add processed images for visualization
+        prediction = self._add_processed_images(prediction, imgs_cpu)
+
+        # Export if requested
+        if export_dir is not None:
+
+            if "gs" in export_format:
+                if infer_gs and "gs_video" not in export_format:
+                    export_format = f"{export_format}-gs_video"
+                if "gs_video" in export_format:
+                    if "gs_video" not in export_kwargs:
+                        export_kwargs["gs_video"] = {}
+                    export_kwargs["gs_video"].update(
+                        {
+                            "extrinsics": render_exts,
+                            "intrinsics": render_ixts,
+                            "out_image_hw": render_hw,
+                        }
+                    )
+            # Add GLB export parameters
+            if "glb" in export_format:
+                if "glb" not in export_kwargs:
+                    export_kwargs["glb"] = {}
+                export_kwargs["glb"].update(
+                    {
+                        "conf_thresh_percentile": conf_thresh_percentile,
+                        "num_max_points": num_max_points,
+                        "show_cameras": show_cameras,
+                    }
+                )
+            # Add Feat_vis export parameters
+            if "feat_vis" in export_format:
+                if "feat_vis" not in export_kwargs:
+                    export_kwargs["feat_vis"] = {}
+                export_kwargs["feat_vis"].update(
+                    {
+                        "fps": feat_vis_fps,
+                    }
+                )
+            # Add COLMAP export parameters
+            if "colmap" in export_format:
+                if "colmap" not in export_kwargs:
+                    export_kwargs["colmap"] = {}
+                export_kwargs["colmap"].update(
+                    {
+                        "image_paths": image,
+                        "conf_thresh_percentile": conf_thresh_percentile,
+                        "process_res_method": process_res_method,
+                    }
+                )
+            self._export_results(prediction, export_format, export_dir, **export_kwargs)
+
+        return prediction
+
+    def _preprocess_inputs(
+        self,
+        image: list[np.ndarray | Image.Image | str],
+        extrinsics: np.ndarray | None = None,
+        intrinsics: np.ndarray | None = None,
+        process_res: int = 504,
+        process_res_method: str = "upper_bound_resize",
+    ) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
+        """Preprocess input images using input processor."""
+        start_time = time.time()
+        imgs_cpu, extrinsics, intrinsics = self.input_processor(
+            image,
+            extrinsics.copy() if extrinsics is not None else None,
+            intrinsics.copy() if intrinsics is not None else None,
+            process_res,
+            process_res_method,
+        )
+        end_time = time.time()
+        logger.info(
+            "Processed Images Done taking",
+            end_time - start_time,
+            "seconds. Shape: ",
+            imgs_cpu.shape,
+        )
+        return imgs_cpu, extrinsics, intrinsics
+
+    def _prepare_model_inputs(
+        self,
+        imgs_cpu: torch.Tensor,
+        extrinsics: torch.Tensor | None,
+        intrinsics: torch.Tensor | None,
+    ) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
+        """Prepare tensors for model input."""
+        device = self._get_model_device()
+
+        # Move images to model device
+        imgs = imgs_cpu.to(device, non_blocking=True)[None].float()
+
+        # Convert camera parameters to tensors
+        ex_t = (
+            extrinsics.to(device, non_blocking=True)[None].float()
+            if extrinsics is not None
+            else None
+        )
+        in_t = (
+            intrinsics.to(device, non_blocking=True)[None].float()
+            if intrinsics is not None
+            else None
+        )
+
+        return imgs, ex_t, in_t
+
+    def _normalize_extrinsics(self, ex_t: torch.Tensor | None) -> torch.Tensor | None:
+        """Normalize extrinsics"""
+        if ex_t is None:
+            return None
+        transform = affine_inverse(ex_t[:, :1])
+        ex_t_norm = ex_t @ transform
+        c2ws = affine_inverse(ex_t_norm)
+        translations = c2ws[..., :3, 3]
+        dists = translations.norm(dim=-1)
+        median_dist = torch.median(dists)
+        median_dist = torch.clamp(median_dist, min=1e-1)
+        ex_t_norm[..., :3, 3] = ex_t_norm[..., :3, 3] / median_dist
+        return ex_t_norm
+
+    def _align_to_input_extrinsics_intrinsics(
+        self,
+        extrinsics: torch.Tensor | None,
+        intrinsics: torch.Tensor | None,
+        prediction: Prediction,
+        align_to_input_ext_scale: bool = True,
+        ransac_view_thresh: int = 10,
+    ) -> Prediction:
+        """Align depth map to input extrinsics"""
+        if extrinsics is None:
+            return prediction
+        from depth_anything_3.utils.pose_align import align_poses_umeyama
+        prediction.intrinsics = intrinsics.numpy()
+        _, _, scale, aligned_extrinsics = align_poses_umeyama(
+            prediction.extrinsics,
+            extrinsics.numpy(),
+            ransac=len(extrinsics) >= ransac_view_thresh,
+            return_aligned=True,
+            random_state=42,
+        )
+        if align_to_input_ext_scale:
+            prediction.extrinsics = extrinsics[..., :3, :].numpy()
+            prediction.depth /= scale
+        else:
+            prediction.extrinsics = aligned_extrinsics
+        return prediction
+
+    def _run_model_forward(
+        self,
+        imgs: torch.Tensor,
+        ex_t: torch.Tensor | None,
+        in_t: torch.Tensor | None,
+        export_feat_layers: Sequence[int] | None = None,
+        infer_gs: bool = False,
+        use_ray_pose: bool = False,
+        ref_view_strategy: str = "saddle_balanced",
+    ) -> dict[str, torch.Tensor]:
+        """Run model forward pass."""
+        device = imgs.device
+        need_sync = device.type == "cuda"
+        if need_sync:
+            torch.cuda.synchronize(device)
+        start_time = time.time()
+        feat_layers = list(export_feat_layers) if export_feat_layers is not None else None
+        output = self.forward(imgs, ex_t, in_t, feat_layers, infer_gs, use_ray_pose, ref_view_strategy)
+        if need_sync:
+            torch.cuda.synchronize(device)
+        end_time = time.time()
+        logger.info(f"Model Forward Pass Done. Time: {end_time - start_time} seconds")
+        return output
+
+    def _convert_to_prediction(self, raw_output: dict[str, torch.Tensor]) -> Prediction:
+        """Convert raw model output to Prediction object."""
+        start_time = time.time()
+        output = self.output_processor(raw_output)
+        end_time = time.time()
+        logger.info(f"Conversion to Prediction Done. Time: {end_time - start_time} seconds")
+        return output
+
+    def _add_processed_images(self, prediction: Prediction, imgs_cpu: torch.Tensor) -> Prediction:
+        """Add processed images to prediction for visualization."""
+        # Convert from (N, 3, H, W) to (N, H, W, 3) and denormalize
+        processed_imgs = imgs_cpu.permute(0, 2, 3, 1).cpu().numpy()  # (N, H, W, 3)
+
+        # Denormalize from ImageNet normalization
+        mean = np.array([0.485, 0.456, 0.406])
+        std = np.array([0.229, 0.224, 0.225])
+        processed_imgs = processed_imgs * std + mean
+        processed_imgs = np.clip(processed_imgs, 0, 1)
+        processed_imgs = (processed_imgs * 255).astype(np.uint8)
+
+        prediction.processed_images = processed_imgs
+        return prediction
+
+    def _export_results(
+        self, prediction: Prediction, export_format: str, export_dir: str, **kwargs
+    ) -> None:
+        """Export results to specified format and directory."""
+        from depth_anything_3.utils.export import export
+        start_time = time.time()
+        export(prediction, export_format, export_dir, **kwargs)
+        end_time = time.time()
+        logger.info(f"Export Results Done. Time: {end_time - start_time} seconds")
+
+    def _get_model_device(self) -> torch.device:
+        """
+        Get the device where the model is located.
+
+        Returns:
+            Device where the model parameters are located
+
+        Raises:
+            ValueError: If no tensors are found in the model
+        """
+        if self.device is not None:
+            return self.device
+
+        # Find device from parameters
+        for param in self.parameters():
+            self.device = param.device
+            return param.device
+
+        # Find device from buffers
+        for buffer in self.buffers():
+            self.device = buffer.device
+            return buffer.device
+
+        raise ValueError("No tensor found in model")

depth_anything_3/app/css_and_html.py

@@ -0,0 +1,594 @@
+# flake8: noqa: E501
+
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+CSS and HTML content for the Depth Anything 3 Gradio application.
+This module contains all the CSS styles and HTML content blocks
+used in the Gradio interface.
+"""
+
+# CSS Styles for the Gradio interface
+GRADIO_CSS = """
+/* Add Font Awesome CDN with all styles including brands and colors */
+@import url('https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css');
+
+/* Add custom styles for colored icons */
+.fa-color-blue {
+    color: #3b82f6;
+}
+
+.fa-color-purple {
+    color: #8b5cf6;
+}
+
+.fa-color-cyan {
+    color: #06b6d4;
+}
+
+.fa-color-green {
+    color: #10b981;
+}
+
+.fa-color-yellow {
+    color: #f59e0b;
+}
+
+.fa-color-red {
+    color: #ef4444;
+}
+
+.link-btn {
+    display: inline-flex;
+    align-items: center;
+    gap: 8px;
+    text-decoration: none;
+    padding: 12px 24px;
+    border-radius: 50px;
+    font-weight: 500;
+    transition: all 0.3s ease;
+}
+
+/* Dark mode tech theme */
+@media (prefers-color-scheme: dark) {
+    html, body {
+        background: #1e293b;
+        color: #ffffff;
+    }
+
+    .gradio-container {
+        background: #1e293b;
+        color: #ffffff;
+    }
+
+    .link-btn {
+        background: rgba(255, 255, 255, 0.2);
+        color: white;
+        backdrop-filter: blur(10px);
+        border: 1px solid rgba(255, 255, 255, 0.3);
+    }
+
+    .link-btn:hover {
+        background: rgba(255, 255, 255, 0.3);
+        transform: translateY(-2px);
+        box-shadow: 0 8px 25px rgba(0, 0, 0, 0.2);
+    }
+
+    .tech-bg {
+        background: linear-gradient(135deg, #0f172a, #1e293b); /* Darker colors */
+        position: relative;
+        overflow: hidden;
+    }
+
+    .tech-bg::before {
+        content: '';
+        position: absolute;
+        top: 0;
+        left: 0;
+        right: 0;
+        bottom: 0;
+        background:
+            radial-gradient(circle at 20% 80%, rgba(59, 130, 246, 0.15) 0%, transparent 50%), /* Reduced opacity */
+            radial-gradient(circle at 80% 20%, rgba(139, 92, 246, 0.15) 0%, transparent 50%), /* Reduced opacity */
+            radial-gradient(circle at 40% 40%, rgba(18, 194, 233, 0.1) 0%, transparent 50%); /* Reduced opacity */
+        animation: techPulse 8s ease-in-out infinite;
+    }
+
+    .gradio-container .panel,
+    .gradio-container .block,
+    .gradio-container .form {
+        background: rgba(0, 0, 0, 0.3);
+        border: 1px solid rgba(59, 130, 246, 0.2);
+        border-radius: 10px;
+    }
+
+    .gradio-container * {
+        color: #ffffff;
+    }
+
+    .gradio-container label {
+        color: #e0e0e0;
+    }
+
+    .gradio-container .markdown {
+        color: #e0e0e0;
+    }
+}
+
+/* Light mode tech theme */
+@media (prefers-color-scheme: light) {
+    html, body {
+        background: #ffffff;
+        color: #1e293b;
+    }
+
+    .gradio-container {
+        background: #ffffff;
+        color: #1e293b;
+    }
+
+    .tech-bg {
+        background: linear-gradient(135deg, #ffffff, #f1f5f9);
+        position: relative;
+        overflow: hidden;
+    }
+
+    .link-btn {
+        background: rgba(59, 130, 246, 0.15);
+        color: var(--body-text-color);
+        border: 1px solid rgba(59, 130, 246, 0.3);
+    }
+
+    .link-btn:hover {
+        background: rgba(59, 130, 246, 0.25);
+        transform: translateY(-2px);
+        box-shadow: 0 8px 25px rgba(59, 130, 246, 0.2);
+    }
+
+    .tech-bg::before {
+        content: '';
+        position: absolute;
+        top: 0;
+        left: 0;
+        right: 0;
+        bottom: 0;
+        background:
+            radial-gradient(circle at 20% 80%, rgba(59, 130, 246, 0.1) 0%, transparent 50%),
+            radial-gradient(circle at 80% 20%, rgba(139, 92, 246, 0.1) 0%, transparent 50%),
+            radial-gradient(circle at 40% 40%, rgba(18, 194, 233, 0.08) 0%, transparent 50%);
+        animation: techPulse 8s ease-in-out infinite;
+    }
+
+    .gradio-container .panel,
+    .gradio-container .block,
+    .gradio-container .form {
+        background: rgba(255, 255, 255, 0.8);
+        border: 1px solid rgba(59, 130, 246, 0.3);
+        border-radius: 10px;
+        box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+    }
+
+    .gradio-container * {
+        color: #1e293b;
+    }
+
+    .gradio-container label {
+        color: #334155;
+    }
+
+    .gradio-container .markdown {
+        color: #334155;
+    }
+}
+
+
+
+
+@keyframes techPulse {
+    0%, 100% { opacity: 0.5; }
+    50% { opacity: 0.8; }
+}
+
+/* Custom log with tech gradient */
+.custom-log * {
+    font-style: italic;
+    font-size: 22px !important;
+    background: linear-gradient(135deg, #3b82f6, #8b5cf6);
+    background-size: 400% 400%;
+    -webkit-background-clip: text;
+    background-clip: text;
+    font-weight: bold !important;
+    color: transparent !important;
+    text-align: center !important;
+    animation: techGradient 3s ease infinite;
+}
+
+@keyframes techGradient {
+    0% { background-position: 0% 50%; }
+    50% { background-position: 100% 50%; }
+    100% { background-position: 0% 50%; }
+}
+
+@keyframes metricPulse {
+    0%, 100% { background-position: 0% 50%; }
+    50% { background-position: 100% 50%; }
+}
+
+@keyframes pointcloudPulse {
+    0%, 100% { background-position: 0% 50%; }
+    50% { background-position: 100% 50%; }
+}
+
+@keyframes camerasPulse {
+    0%, 100% { background-position: 0% 50%; }
+    50% { background-position: 100% 50%; }
+}
+
+@keyframes gaussiansPulse {
+    0%, 100% { background-position: 0% 50%; }
+    50% { background-position: 100% 50%; }
+}
+
+/* Special colors for key terms - Global styles */
+.metric-text {
+    background: linear-gradient(45deg, #ff6b6b, #ff8e53, #ff6b6b);
+    background-size: 200% 200%;
+    -webkit-background-clip: text;
+    background-clip: text;
+    color: transparent !important;
+    animation: metricPulse 2s ease-in-out infinite;
+    font-weight: 700;
+    text-shadow: 0 0 10px rgba(255, 107, 107, 0.5);
+}
+
+.pointcloud-text {
+    background: linear-gradient(45deg, #4ecdc4, #44a08d, #4ecdc4);
+    background-size: 200% 200%;
+    -webkit-background-clip: text;
+    background-clip: text;
+    color: transparent !important;
+    animation: pointcloudPulse 2.5s ease-in-out infinite;
+    font-weight: 700;
+    text-shadow: 0 0 10px rgba(78, 205, 196, 0.5);
+}
+
+.cameras-text {
+    background: linear-gradient(45deg, #667eea, #764ba2, #667eea);
+    background-size: 200% 200%;
+    -webkit-background-clip: text;
+    background-clip: text;
+    color: transparent !important;
+    animation: camerasPulse 3s ease-in-out infinite;
+    font-weight: 700;
+    text-shadow: 0 0 10px rgba(102, 126, 234, 0.5);
+}
+
+.gaussians-text {
+    background: linear-gradient(45deg, #f093fb, #f5576c, #f093fb);
+    background-size: 200% 200%;
+    -webkit-background-clip: text;
+    background-clip: text;
+    color: transparent !important;
+    animation: gaussiansPulse 2.2s ease-in-out infinite;
+    font-weight: 700;
+    text-shadow: 0 0 10px rgba(240, 147, 251, 0.5);
+}
+
+.example-log * {
+    font-style: italic;
+    font-size: 16px !important;
+    background: linear-gradient(135deg, #3b82f6, #8b5cf6);
+    -webkit-background-clip: text;
+    background-clip: text;
+    color: transparent !important;
+}
+
+#my_radio .wrap {
+    display: flex;
+    flex-wrap: nowrap;
+    justify-content: center;
+    align-items: center;
+}
+
+#my_radio .wrap label {
+    display: flex;
+    width: 50%;
+    justify-content: center;
+    align-items: center;
+    margin: 0;
+    padding: 10px 0;
+    box-sizing: border-box;
+}
+
+/* Align navigation buttons with dropdown bottom */
+.navigation-row {
+    display: flex !important;
+    align-items: flex-end !important;
+    gap: 8px !important;
+}
+
+.navigation-row > div:nth-child(1),
+.navigation-row > div:nth-child(3) {
+    align-self: flex-end !important;
+}
+
+.navigation-row > div:nth-child(2) {
+    flex: 1 !important;
+}
+
+/* Make thumbnails clickable with pointer cursor */
+.clickable-thumbnail img {
+    cursor: pointer !important;
+}
+
+.clickable-thumbnail:hover img {
+    cursor: pointer !important;
+    opacity: 0.8;
+    transition: opacity 0.3s ease;
+}
+
+/* Make thumbnail containers narrower horizontally */
+.clickable-thumbnail {
+    padding: 5px 2px !important;
+    margin: 0 2px !important;
+}
+
+.clickable-thumbnail .image-container {
+    margin: 0 !important;
+    padding: 0 !important;
+}
+
+.scene-info {
+    text-align: center !important;
+    padding: 5px 2px !important;
+    margin: 0 !important;
+}
+"""
+
+
+def get_header_html(logo_base64=None):
+    """
+    Generate the main header HTML with logo and title.
+
+    Args:
+        logo_base64 (str, optional): Base64 encoded logo image
+
+    Returns:
+        str: HTML string for the header
+    """
+    return """
+    <div class="tech-bg" style="text-align: center; margin-bottom: 5px; padding: 40px 20px; border-radius: 15px; position: relative; overflow: hidden;">
+        <div style="position: relative; z-index: 2;">
+            <h1 style="margin: 0; font-size: 3.5em; font-weight: 700;
+                background: linear-gradient(135deg, #3b82f6, #8b5cf6);
+                background-size: 400% 400%;
+                -webkit-background-clip: text;
+                background-clip: text;
+                color: transparent;
+                animation: techGradient 3s ease infinite;
+                text-shadow: 0 0 30px rgba(59, 130, 246, 0.5);
+                letter-spacing: 2px;">
+                Depth Anything 3
+            </h1>
+            <p style="margin: 15px 0 0 0; font-size: 2.16em; font-weight: 300;" class="header-subtitle">
+                Recovering the Visual Space from Any Views
+            </p>
+            <div style="margin-top: 20px;">
+                <!-- Revert buttons to original inline styles -->
+                <a href="https://depth-anything-3.github.io" target="_blank" class="link-btn">
+                    <i class="fas fa-globe" style="margin-right: 8px;"></i> Project Page
+                </a>
+                <a href="https://arxiv.org/abs/2406.09414" target="_blank" class="link-btn">
+                    <i class="fas fa-file-pdf" style="margin-right: 8px;"></i> Paper
+                </a>
+                <a href="https://github.com/ByteDance-Seed/Depth-Anything-3" target="_blank" class="link-btn">
+                    <i class="fab fa-github" style="margin-right: 8px;"></i> Code
+                </a>
+            </div>
+        </div>
+    </div>
+
+    <style>
+        /* Ensure tech-bg class is properly applied in dark mode */
+        @media (prefers-color-scheme: dark) {
+            .header-subtitle {
+                color: #cbd5e1;
+            }
+            /* Increase priority to ensure background color is properly applied */
+            .tech-bg {
+                background: linear-gradient(135deg, #0f172a, #1e293b) !important;
+            }
+        }
+
+        @media (prefers-color-scheme: light) {
+            .header-subtitle {
+                color: #475569;
+            }
+            /* Also add explicit background color for light mode */
+            .tech-bg {
+                background: linear-gradient(135deg, rgba(59, 130, 246, 0.1) 0%, rgba(139, 92, 246, 0.1) 100%) !important;
+            }
+        }
+    </style>
+    """
+
+
+def get_description_html():
+    """
+    Generate the main description and getting started HTML.
+
+    Returns:
+        str: HTML string for the description
+    """
+    return """
+    <div class="description-container" style="padding: 25px; border-radius: 15px; margin: 0 0 20px 0;">
+        <h2 class="description-title" style="margin-top: 0; font-size: 1.6em; text-align: center;">
+            <i class="fas fa-bullseye fa-color-red" style="margin-right: 8px;"></i> What This Demo Does
+        </h2>
+        <div class="description-content" style="padding: 20px; border-radius: 10px; margin: 15px 0; text-align: center;">
+            <p class="description-main" style="line-height: 1.6; margin: 0; font-size: 1.45em;">
+                <strong>Upload images or videos</strong> → <strong>Get <span class="metric-text">Metric</span> <span class="pointcloud-text">Point Clouds</span>, <span class="cameras-text">Cameras</span> and <span class="gaussians-text">Novel Views</span></strong> → <strong>Explore in 3D</strong>
+            </p>
+        </div>
+
+        <div style="text-align: center; margin-top: 15px;">
+            <p class="description-tip" style="font-style: italic; margin: 0;">
+                <i class="fas fa-lightbulb fa-color-yellow" style="margin-right: 8px;"></i> <strong>Tip:</strong> Landscape-oriented images or videos are preferred for best 3D recovering.
+            </p>
+        </div>
+    </div>
+
+    <style>
+        @media (prefers-color-scheme: dark) {
+            .description-container {
+                background: linear-gradient(135deg, rgba(59, 130, 246, 0.1) 0%, rgba(139, 92, 246, 0.1) 100%);
+                border: 1px solid rgba(59, 130, 246, 0.2);
+            }
+            .description-title { color: #3b82f6; }
+            .description-content { background: rgba(0, 0, 0, 0.3); }
+            .description-main { color: #e0e0e0; }
+            .description-text { color: #cbd5e1; }
+            .description-tip { color: #cbd5e1; }
+        }
+
+        @media (prefers-color-scheme: light) {
+            .description-container {
+                background: linear-gradient(135deg, rgba(59, 130, 246, 0.05) 0%, rgba(139, 92, 246, 0.05) 100%);
+                border: 1px solid rgba(59, 130, 246, 0.3);
+            }
+            .description-title { color: #3b82f6; }
+            .description-content { background: transparent; }
+            .description-main { color: #1e293b; }
+            .description-text { color: #475569; }
+            .description-tip { color: #475569; }
+        }
+    </style>
+    """
+
+
+def get_acknowledgements_html():
+    """
+    Generate the acknowledgements section HTML.
+
+    Returns:
+        str: HTML string for the acknowledgements
+    """
+    return """
+    <div style="background: linear-gradient(135deg, rgba(59, 130, 246, 0.1) 0%, rgba(139, 92, 246, 0.1) 100%);
+                padding: 25px; border-radius: 15px; margin: 20px 0; border: 1px solid rgba(59, 130, 246, 0.2);">
+        <h3 style="color: #3b82f6; margin-top: 0; text-align: center; font-size: 1.4em;">
+            <i class="fas fa-trophy fa-color-yellow" style="margin-right: 8px;"></i> Research Credits & Acknowledgments
+        </h3>
+
+        <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 20px; margin: 15px 0;">
+            <!-- Original Research Section (Left) -->
+            <div style="text-align: center;">
+                <h4 style="color: #8b5cf6; margin: 10px 0;"><i class="fas fa-flask fa-color-green" style="margin-right: 8px;"></i> Original Research</h4>
+                <p style="color: #e0e0e0; margin: 5px 0;">
+                    <a href="https://depth-anything-3.github.io" target="_blank"
+                       style="color: #3b82f6; text-decoration: none; font-weight: 600;">
+                        Depth Anything 3
+                    </a>
+                </p>
+            </div>
+
+            <!-- Previous Versions Section (Right) -->
+            <div style="text-align: center;">
+                <h4 style="color: #8b5cf6; margin: 10px 0;"><i class="fas fa-history fa-color-blue" style="margin-right: 8px;"></i> Previous Versions</h4>
+                <div style="display: flex; flex-direction: row; gap: 15px; justify-content: center; align-items: center;">
+                    <p style="color: #e0e0e0; margin: 0;">
+                        <a href="https://huggingface.co/spaces/LiheYoung/Depth-Anything" target="_blank"
+                           style="color: #3b82f6; text-decoration: none; font-weight: 600;">
+                            Depth-Anything
+                        </a>
+                    </p>
+                    <span style="color: #e0e0e0;">•</span>
+                    <p style="color: #e0e0e0; margin: 0;">
+                        <a href="https://huggingface.co/spaces/depth-anything/Depth-Anything-V2" target="_blank"
+                           style="color: #3b82f6; text-decoration: none; font-weight: 600;">
+                            Depth-Anything-V2
+                        </a>
+                    </p>
+                </div>
+            </div>
+        </div>
+
+        <!-- HF Demo Adapted from - Centered at the bottom of the whole block -->
+        <div style="margin-top: 20px; padding-top: 15px; border-top: 1px solid rgba(59, 130, 246, 0.3); text-align: center;">
+            <p style="color: #a0a0a0; font-size: 0.9em; margin: 0;">
+                <i class="fas fa-code-branch fa-color-gray" style="margin-right: 5px;"></i> HF demo adapted from <a href="https://huggingface.co/spaces/facebook/map-anything" target="_blank" style="color: inherit; text-decoration: none;">Map Anything</a>
+            </p>
+        </div>
+    </div>
+    """
+
+
+def get_gradio_theme():
+    """
+    Get the configured Gradio theme with adaptive tech colors.
+
+    Returns:
+        gr.themes.Base: Configured Gradio theme
+    """
+    import gradio as gr
+
+    return gr.themes.Base(
+        primary_hue=gr.themes.Color(
+            c50="#eff6ff",
+            c100="#dbeafe",
+            c200="#bfdbfe",
+            c300="#93c5fd",
+            c400="#60a5fa",
+            c500="#3b82f6",
+            c600="#2563eb",
+            c700="#1d4ed8",
+            c800="#1e40af",
+            c900="#1e3a8a",
+            c950="#172554",
+        ),
+        secondary_hue=gr.themes.Color(
+            c50="#f5f3ff",
+            c100="#ede9fe",
+            c200="#ddd6fe",
+            c300="#c4b5fd",
+            c400="#a78bfa",
+            c500="#8b5cf6",
+            c600="#7c3aed",
+            c700="#6d28d9",
+            c800="#5b21b6",
+            c900="#4c1d95",
+            c950="#2e1065",
+        ),
+        neutral_hue=gr.themes.Color(
+            c50="#f8fafc",
+            c100="#f1f5f9",
+            c200="#e2e8f0",
+            c300="#cbd5e1",
+            c400="#94a3b8",
+            c500="#64748b",
+            c600="#475569",
+            c700="#334155",
+            c800="#1e293b",
+            c900="#0f172a",
+            c950="#020617",
+        ),
+    )
+
+
+# Measure tab instructions HTML
+MEASURE_INSTRUCTIONS_HTML = """
+### Click points on the image to compute distance.
+> <i class="fas fa-triangle-exclamation fa-color-red" style="margin-right: 5px;"></i> Metric scale estimation is difficult on aerial/drone images.
+"""

depth_anything_3/app/gradio_app.py

@@ -0,0 +1,724 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Refactored Gradio App for Depth Anything 3.
+
+This is the main application file that orchestrates all components.
+The original functionality has been split into modular components for better maintainability.
+"""
+
+import argparse
+import os
+from typing import Any, Dict, List
+import gradio as gr
+
+from depth_anything_3.app.css_and_html import GRADIO_CSS, get_gradio_theme
+from depth_anything_3.app.modules.event_handlers import EventHandlers
+from depth_anything_3.app.modules.ui_components import UIComponents
+
+# Set environment variables
+os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
+
+
+class DepthAnything3App:
+    """
+    Main application class for Depth Anything 3 Gradio app.
+    """
+
+    def __init__(self, model_dir: str = None, workspace_dir: str = None, gallery_dir: str = None):
+        """
+        Initialize the application.
+
+        Args:
+            model_dir: Path to the model directory
+            workspace_dir: Path to the workspace directory
+            gallery_dir: Path to the gallery directory
+        """
+        self.model_dir = model_dir
+        self.workspace_dir = workspace_dir
+        self.gallery_dir = gallery_dir
+
+        # Set environment variables for directories
+        if self.model_dir:
+            os.environ["DA3_MODEL_DIR"] = self.model_dir
+        if self.workspace_dir:
+            os.environ["DA3_WORKSPACE_DIR"] = self.workspace_dir
+        if self.gallery_dir:
+            os.environ["DA3_GALLERY_DIR"] = self.gallery_dir
+
+        self.event_handlers = EventHandlers()
+        self.ui_components = UIComponents()
+
+    def cache_examples(
+        self,
+        show_cam: bool = True,
+        filter_black_bg: bool = False,
+        filter_white_bg: bool = False,
+        save_percentage: float = 20.0,
+        num_max_points: int = 1000,
+        cache_gs_tag: str = "",
+        gs_trj_mode: str = "smooth",
+        gs_video_quality: str = "low",
+    ) -> None:
+        """
+        Pre-cache all example scenes at startup.
+
+        Args:
+            show_cam: Whether to show camera in visualization
+            filter_black_bg: Whether to filter black background
+            filter_white_bg: Whether to filter white background
+            save_percentage: Filter percentage for point cloud
+            num_max_points: Maximum number of points
+            cache_gs_tag: Tag to match scene names for high-res+3DGS caching (e.g., "dl3dv")
+            gs_trj_mode: Trajectory mode for 3DGS
+            gs_video_quality: Video quality for 3DGS
+        """
+        from depth_anything_3.app.modules.utils import get_scene_info
+
+        examples_dir = os.path.join(self.workspace_dir, "examples")
+        if not os.path.exists(examples_dir):
+            print(f"Examples directory not found: {examples_dir}")
+            return
+
+        scenes = get_scene_info(examples_dir)
+        if not scenes:
+            print("No example scenes found to cache.")
+            return
+
+        print(f"\n{'='*60}")
+        print(f"Caching {len(scenes)} example scenes...")
+        print(f"{'='*60}\n")
+
+        for i, scene in enumerate(scenes, 1):
+            scene_name = scene["name"]
+
+            # Check if scene name matches the gs tag for high-res+3DGS caching
+            use_high_res_gs = cache_gs_tag and cache_gs_tag.lower() in scene_name.lower()
+
+            if use_high_res_gs:
+                print(f"[{i}/{len(scenes)}] Caching scene: {scene_name} (HIGH-RES + 3DGS)")
+                print(f"  - Number of images: {scene['num_images']}")
+                print(f"  - Matched tag: '{cache_gs_tag}' - using high_res + 3DGS")
+            else:
+                print(f"[{i}/{len(scenes)}] Caching scene: {scene_name} (LOW-RES)")
+                print(f"  - Number of images: {scene['num_images']}")
+
+            try:
+                # Load example scene
+                _, target_dir, _, _, _, _, _, _, _ = self.event_handlers.load_example_scene(
+                    scene_name
+                )
+
+                if target_dir and target_dir != "None":
+                    # Run reconstruction with appropriate settings
+                    print("  - Running reconstruction...")
+                    result = self.event_handlers.gradio_demo(
+                        target_dir=target_dir,
+                        show_cam=show_cam,
+                        filter_black_bg=filter_black_bg,
+                        filter_white_bg=filter_white_bg,
+                        process_res_method="high_res" if use_high_res_gs else "low_res",
+                        save_percentage=save_percentage,
+                        num_max_points=num_max_points,
+                        infer_gs=use_high_res_gs,
+                        ref_view_strategy="saddle_balanced",
+                        gs_trj_mode=gs_trj_mode,
+                        gs_video_quality=gs_video_quality,
+                    )
+
+                    # Check if successful
+                    if result[0] is not None:  # reconstruction_output
+                        print(f"  ✓ Scene '{scene_name}' cached successfully")
+                    else:
+                        print(f"  ✗ Scene '{scene_name}' caching failed: {result[1]}")
+                else:
+                    print(f"  ✗ Scene '{scene_name}' loading failed")
+
+            except Exception as e:
+                print(f"  ✗ Error caching scene '{scene_name}': {str(e)}")
+
+            print()
+
+        print("=" * 60)
+        print("Example scene caching completed!")
+        print("=" * 60 + "\n")
+
+    def create_app(self) -> gr.Blocks:
+        """
+        Create and configure the Gradio application.
+
+        Returns:
+            Configured Gradio Blocks interface
+        """
+
+        # Initialize theme
+        def get_theme():
+            return get_gradio_theme()
+
+        with gr.Blocks(theme=get_theme(), css=GRADIO_CSS) as demo:
+            # State variables for the tabbed interface
+            is_example = gr.Textbox(label="is_example", visible=False, value="None")
+            processed_data_state = gr.State(value=None)
+            measure_points_state = gr.State(value=[])
+            selected_image_index_state = gr.State(value=0)  # Track selected image index
+            # current_view_index = gr.State(value=0)  # noqa: F841 Track current view index
+
+            # Header and description
+            self.ui_components.create_header_section()
+            self.ui_components.create_description_section()
+
+            target_dir_output = gr.Textbox(label="Target Dir", visible=False, value="None")
+
+            # Main content area
+            with gr.Row():
+                with gr.Column(scale=2):
+                    # Upload section
+                    (
+                        input_video,
+                        s_time_interval,
+                        input_images,
+                        image_gallery,
+                    ) = self.ui_components.create_upload_section()
+
+                with gr.Column(scale=4):
+                    with gr.Column():
+                        # gr.Markdown("**Metric 3D Reconstruction (Point Cloud and Camera Poses)**")
+                        # Reconstruction control section (buttons) - moved below tabs
+
+                        log_output = gr.Markdown(
+                            "Please upload a video or images, then click Reconstruct.",
+                            elem_classes=["custom-log"],
+                        )
+
+                        # Tabbed interface
+                        with gr.Tabs():
+                            with gr.Tab("Point Cloud & Cameras"):
+                                reconstruction_output = (
+                                    self.ui_components.create_3d_viewer_section()
+                                )
+
+                            with gr.Tab("Metric Depth"):
+                                (
+                                    prev_measure_btn,
+                                    measure_view_selector,
+                                    next_measure_btn,
+                                    measure_image,
+                                    measure_depth_image,
+                                    measure_text,
+                                ) = self.ui_components.create_measure_section()
+
+                            with gr.Tab("3DGS Rendered Novel Views"):
+                                gs_video, gs_info = self.ui_components.create_nvs_video()
+
+                        # Inference control section (before inference)
+                        (process_res_method_dropdown, infer_gs, ref_view_strategy_dropdown) = (
+                            self.ui_components.create_inference_control_section()
+                        )
+
+                        # Display control section - includes 3DGS options, buttons, and Visualization Options  # noqa: E501
+                        (
+                            show_cam,
+                            filter_black_bg,
+                            filter_white_bg,
+                            save_percentage,
+                            num_max_points,
+                            gs_trj_mode,
+                            gs_video_quality,
+                            submit_btn,
+                            clear_btn,
+                        ) = self.ui_components.create_display_control_section()
+
+                        # bind visibility of gs_trj_mode to infer_gs
+                        infer_gs.change(
+                            fn=lambda checked: (
+                                gr.update(visible=checked),
+                                gr.update(visible=checked),
+                                gr.update(visible=checked),
+                                gr.update(visible=(not checked)),
+                            ),
+                            inputs=infer_gs,
+                            outputs=[gs_trj_mode, gs_video_quality, gs_video, gs_info],
+                        )
+
+            # Example scenes section
+            gr.Markdown("## Example Scenes")
+
+            scenes = self.ui_components.create_example_scenes_section()
+            scene_components = self.ui_components.create_example_scene_grid(scenes)
+
+            # Set up event handlers
+            self._setup_event_handlers(
+                demo,
+                is_example,
+                processed_data_state,
+                measure_points_state,
+                target_dir_output,
+                input_video,
+                input_images,
+                s_time_interval,
+                image_gallery,
+                reconstruction_output,
+                log_output,
+                show_cam,
+                filter_black_bg,
+                filter_white_bg,
+                process_res_method_dropdown,
+                save_percentage,
+                submit_btn,
+                clear_btn,
+                num_max_points,
+                infer_gs,
+                ref_view_strategy_dropdown,
+                selected_image_index_state,
+                measure_view_selector,
+                measure_image,
+                measure_depth_image,
+                measure_text,
+                prev_measure_btn,
+                next_measure_btn,
+                scenes,
+                scene_components,
+                gs_video,
+                gs_info,
+                gs_trj_mode,
+                gs_video_quality,
+            )
+
+            # Acknowledgements
+            self.ui_components.create_acknowledgements_section()
+
+        return demo
+
+    def _setup_event_handlers(
+        self,
+        demo: gr.Blocks,
+        is_example: gr.Textbox,
+        processed_data_state: gr.State,
+        measure_points_state: gr.State,
+        target_dir_output: gr.Textbox,
+        input_video: gr.Video,
+        input_images: gr.File,
+        s_time_interval: gr.Slider,
+        image_gallery: gr.Gallery,
+        reconstruction_output: gr.Model3D,
+        log_output: gr.Markdown,
+        show_cam: gr.Checkbox,
+        filter_black_bg: gr.Checkbox,
+        filter_white_bg: gr.Checkbox,
+        process_res_method_dropdown: gr.Dropdown,
+        save_percentage: gr.Slider,
+        submit_btn: gr.Button,
+        clear_btn: gr.ClearButton,
+        num_max_points: gr.Slider,
+        infer_gs: gr.Checkbox,
+        ref_view_strategy_dropdown: gr.Dropdown,
+        selected_image_index_state: gr.State,
+        measure_view_selector: gr.Dropdown,
+        measure_image: gr.Image,
+        measure_depth_image: gr.Image,
+        measure_text: gr.Markdown,
+        prev_measure_btn: gr.Button,
+        next_measure_btn: gr.Button,
+        scenes: List[Dict[str, Any]],
+        scene_components: List[gr.Image],
+        gs_video: gr.Video,
+        gs_info: gr.Markdown,
+        gs_trj_mode: gr.Dropdown,
+        gs_video_quality: gr.Dropdown,
+    ) -> None:
+        """
+        Set up all event handlers for the application.
+
+        Args:
+            demo: Gradio Blocks interface
+            All other arguments: Gradio components to connect
+        """
+        # Configure clear button
+        clear_btn.add(
+            [
+                input_video,
+                input_images,
+                reconstruction_output,
+                log_output,
+                target_dir_output,
+                image_gallery,
+                gs_video,
+            ]
+        )
+
+        # Main reconstruction button
+        submit_btn.click(
+            fn=self.event_handlers.clear_fields, inputs=[], outputs=[reconstruction_output]
+        ).then(fn=self.event_handlers.update_log, inputs=[], outputs=[log_output]).then(
+            fn=self.event_handlers.gradio_demo,
+            inputs=[
+                target_dir_output,
+                show_cam,
+                filter_black_bg,
+                filter_white_bg,
+                process_res_method_dropdown,
+                save_percentage,
+                # pass num_max_points
+                num_max_points,
+                infer_gs,
+                ref_view_strategy_dropdown,
+                gs_trj_mode,
+                gs_video_quality,
+            ],
+            outputs=[
+                reconstruction_output,
+                log_output,
+                processed_data_state,
+                measure_image,
+                measure_depth_image,
+                measure_text,
+                measure_view_selector,
+                gs_video,
+                gs_video,  # gs_video visibility
+                gs_info,  # gs_info visibility
+            ],
+        ).then(
+            fn=lambda: "False",
+            inputs=[],
+            outputs=[is_example],  # set is_example to "False"
+        )
+
+        # Real-time visualization updates
+        self._setup_visualization_handlers(
+            show_cam,
+            filter_black_bg,
+            filter_white_bg,
+            process_res_method_dropdown,
+            target_dir_output,
+            is_example,
+            reconstruction_output,
+            log_output,
+        )
+
+        # File upload handlers
+        input_video.change(
+            fn=self.event_handlers.handle_uploads,
+            inputs=[input_video, input_images, s_time_interval],
+            outputs=[reconstruction_output, target_dir_output, image_gallery, log_output],
+        )
+        input_images.change(
+            fn=self.event_handlers.handle_uploads,
+            inputs=[input_video, input_images, s_time_interval],
+            outputs=[reconstruction_output, target_dir_output, image_gallery, log_output],
+        )
+
+        # Navigation handlers
+        self._setup_navigation_handlers(
+            prev_measure_btn,
+            next_measure_btn,
+            measure_view_selector,
+            measure_image,
+            measure_depth_image,
+            measure_points_state,
+            processed_data_state,
+        )
+
+        # Measurement handler
+        measure_image.select(
+            fn=self.event_handlers.measure,
+            inputs=[processed_data_state, measure_points_state, measure_view_selector],
+            outputs=[measure_image, measure_depth_image, measure_points_state, measure_text],
+        )
+
+        # Example scene handlers
+        self._setup_example_scene_handlers(
+            scenes,
+            scene_components,
+            reconstruction_output,
+            target_dir_output,
+            image_gallery,
+            log_output,
+            is_example,
+            processed_data_state,
+            measure_view_selector,
+            measure_image,
+            measure_depth_image,
+            gs_video,
+            gs_info,
+        )
+
+    def _setup_visualization_handlers(
+        self,
+        show_cam: gr.Checkbox,
+        filter_black_bg: gr.Checkbox,
+        filter_white_bg: gr.Checkbox,
+        process_res_method_dropdown: gr.Dropdown,
+        target_dir_output: gr.Textbox,
+        is_example: gr.Textbox,
+        reconstruction_output: gr.Model3D,
+        log_output: gr.Markdown,
+    ) -> None:
+        """Set up visualization update handlers."""
+        # Common inputs for visualization updates
+        viz_inputs = [
+            target_dir_output,
+            show_cam,
+            is_example,
+            filter_black_bg,
+            filter_white_bg,
+            process_res_method_dropdown,
+        ]
+
+        # Set up change handlers for all visualization controls
+        for component in [show_cam, filter_black_bg, filter_white_bg]:
+            component.change(
+                fn=self.event_handlers.update_visualization,
+                inputs=viz_inputs,
+                outputs=[reconstruction_output, log_output],
+            )
+
+    def _setup_navigation_handlers(
+        self,
+        prev_measure_btn: gr.Button,
+        next_measure_btn: gr.Button,
+        measure_view_selector: gr.Dropdown,
+        measure_image: gr.Image,
+        measure_depth_image: gr.Image,
+        measure_points_state: gr.State,
+        processed_data_state: gr.State,
+    ) -> None:
+        """Set up navigation handlers for measure tab."""
+        # Measure tab navigation
+        prev_measure_btn.click(
+            fn=lambda processed_data, current_selector: self.event_handlers.navigate_measure_view(
+                processed_data, current_selector, -1
+            ),
+            inputs=[processed_data_state, measure_view_selector],
+            outputs=[
+                measure_view_selector,
+                measure_image,
+                measure_depth_image,
+                measure_points_state,
+            ],
+        )
+
+        next_measure_btn.click(
+            fn=lambda processed_data, current_selector: self.event_handlers.navigate_measure_view(
+                processed_data, current_selector, 1
+            ),
+            inputs=[processed_data_state, measure_view_selector],
+            outputs=[
+                measure_view_selector,
+                measure_image,
+                measure_depth_image,
+                measure_points_state,
+            ],
+        )
+
+        measure_view_selector.change(
+            fn=lambda processed_data, selector_value: (
+                self.event_handlers.update_measure_view(
+                    processed_data, int(selector_value.split()[1]) - 1
+                )
+                if selector_value
+                else (None, None, [])
+            ),
+            inputs=[processed_data_state, measure_view_selector],
+            outputs=[measure_image, measure_depth_image, measure_points_state],
+        )
+
+    def _setup_example_scene_handlers(
+        self,
+        scenes: List[Dict[str, Any]],
+        scene_components: List[gr.Image],
+        reconstruction_output: gr.Model3D,
+        target_dir_output: gr.Textbox,
+        image_gallery: gr.Gallery,
+        log_output: gr.Markdown,
+        is_example: gr.Textbox,
+        processed_data_state: gr.State,
+        measure_view_selector: gr.Dropdown,
+        measure_image: gr.Image,
+        measure_depth_image: gr.Image,
+        gs_video: gr.Video,
+        gs_info: gr.Markdown,
+    ) -> None:
+        """Set up example scene handlers."""
+
+        def load_and_update_measure(name):
+            result = self.event_handlers.load_example_scene(name)
+            # result = (reconstruction_output, target_dir, image_paths, log_message, processed_data, measure_view_selector, gs_video, gs_video_vis, gs_info_vis)  # noqa: E501
+
+            # Update measure view if processed_data is available
+            measure_img = None
+            measure_depth = None
+            if result[4] is not None:  # processed_data exists
+                measure_img, measure_depth, _ = (
+                    self.event_handlers.visualization_handler.update_measure_view(result[4], 0)
+                )
+
+            return result + ("True", measure_img, measure_depth)
+
+        for i, scene in enumerate(scenes):
+            if i < len(scene_components):
+                scene_components[i].select(
+                    fn=lambda name=scene["name"]: load_and_update_measure(name),
+                    outputs=[
+                        reconstruction_output,
+                        target_dir_output,
+                        image_gallery,
+                        log_output,
+                        processed_data_state,
+                        measure_view_selector,
+                        gs_video,
+                        gs_video,  # gs_video_visibility
+                        gs_info,  # gs_info_visibility
+                        is_example,
+                        measure_image,
+                        measure_depth_image,
+                    ],
+                )
+
+    def launch(self, host: str = "127.0.0.1", port: int = 7860, **kwargs) -> None:
+        """
+        Launch the application.
+
+        Args:
+            host: Host address to bind to
+            port: Port number to bind to
+            **kwargs: Additional arguments for demo.launch()
+        """
+        demo = self.create_app()
+        demo.queue(max_size=20).launch(
+            show_error=True, ssr_mode=False, server_name=host, server_port=port, **kwargs
+        )
+
+
+def main():
+    """Main function to run the application."""
+    parser = argparse.ArgumentParser(
+        description="Depth Anything 3 Gradio Application",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Basic usage
+  python gradio_app.py --help
+  python gradio_app.py --host 0.0.0.0 --port 8080
+  python gradio_app.py --model-dir /path/to/model --workspace-dir /path/to/workspace
+
+  # Cache examples at startup (all low-res)
+  python gradio_app.py --cache-examples
+
+  # Cache with selective high-res+3DGS for scenes matching tag
+  python gradio_app.py --cache-examples --cache-gs-tag dl3dv
+  # This will use high-res + 3DGS for scenes containing "dl3dv" in their name,
+  # and low-res only for other scenes
+        """,
+    )
+
+    # Server configuration
+    parser.add_argument(
+        "--host", default="127.0.0.1", help="Host address to bind to (default: 127.0.0.1)"
+    )
+    parser.add_argument(
+        "--port", type=int, default=7860, help="Port number to bind to (default: 7860)"
+    )
+
+    # Directory configuration
+    parser.add_argument(
+        "--model-dir",
+        default="depth-anything/DA3NESTED-GIANT-LARGE",
+        help="Path to the model directory (default: depth-anything/DA3NESTED-GIANT-LARGE)",
+    )
+    parser.add_argument(
+        "--workspace-dir",
+        default="workspace/gradio",  # noqa: E501
+        help="Path to the workspace directory (default: workspace/gradio)",  # noqa: E501
+    )
+    parser.add_argument(
+        "--gallery-dir",
+        default="workspace/gallery",
+        help="Path to the gallery directory (default: workspace/gallery)",  # noqa: E501
+    )
+
+    # Additional Gradio options
+    parser.add_argument("--share", action="store_true", help="Create a public link for the app")
+    parser.add_argument("--debug", action="store_true", help="Enable debug mode")
+
+    # Example caching options
+    parser.add_argument(
+        "--cache-examples",
+        action="store_true",
+        help="Pre-cache all example scenes at startup for faster loading",
+    )
+    parser.add_argument(
+        "--cache-gs-tag",
+        type=str,
+        default="",
+        help="Tag to match scene names for high-res+3DGS caching (e.g., 'dl3dv'). Scenes containing this tag will use high_res and infer_gs=True; others will use low_res only.",  # noqa: E501
+    )
+
+    args = parser.parse_args()
+
+    # Create directories if they don't exist
+    os.makedirs(args.workspace_dir, exist_ok=True)
+    os.makedirs(args.gallery_dir, exist_ok=True)
+
+    # Initialize and launch the application
+    app = DepthAnything3App(
+        model_dir=args.model_dir, workspace_dir=args.workspace_dir, gallery_dir=args.gallery_dir
+    )
+
+    # Prepare launch arguments
+    launch_kwargs = {"share": args.share, "debug": args.debug}
+
+    print("Starting Depth Anything 3 Gradio App...")
+    print(f"Host: {args.host}")
+    print(f"Port: {args.port}")
+    print(f"Model Directory: {args.model_dir}")
+    print(f"Workspace Directory: {args.workspace_dir}")
+    print(f"Gallery Directory: {args.gallery_dir}")
+    print(f"Share: {args.share}")
+    print(f"Debug: {args.debug}")
+    print(f"Cache Examples: {args.cache_examples}")
+    if args.cache_examples:
+        if args.cache_gs_tag:
+            print(
+                f"Cache GS Tag: '{args.cache_gs_tag}' (scenes matching this tag will use high-res + 3DGS)"  # noqa: E501
+            )  # noqa: E501
+        else:
+            print("Cache GS Tag: None (all scenes will use low-res only)")
+
+    # Pre-cache examples if requested
+    if args.cache_examples:
+        print("\n" + "=" * 60)
+        print("Pre-caching mode enabled")
+        if args.cache_gs_tag:
+            print(f"Scenes containing '{args.cache_gs_tag}' will use HIGH-RES + 3DGS")
+            print("Other scenes will use LOW-RES only")
+        else:
+            print("All scenes will use LOW-RES only")
+        print("=" * 60)
+        app.cache_examples(
+            show_cam=True,
+            filter_black_bg=False,
+            filter_white_bg=False,
+            save_percentage=5.0,
+            num_max_points=1000,
+            cache_gs_tag=args.cache_gs_tag,
+            gs_trj_mode="smooth",
+            gs_video_quality="low",
+        )
+
+    app.launch(host=args.host, port=args.port, **launch_kwargs)
+
+
+if __name__ == "__main__":
+    main()

depth_anything_3/app/modules/__init__.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Modules package for Depth Anything 3 Gradio app.
+
+This package contains all the modular components for the Gradio application.
+"""
+
+from depth_anything_3.app.modules.event_handlers import EventHandlers
+from depth_anything_3.app.modules.file_handlers import FileHandler
+from depth_anything_3.app.modules.model_inference import ModelInference
+from depth_anything_3.app.modules.ui_components import UIComponents
+from depth_anything_3.app.modules.utils import (
+    create_depth_visualization,
+    get_logo_base64,
+    get_scene_info,
+    save_to_gallery_func,
+)
+from depth_anything_3.app.modules.visualization import VisualizationHandler
+
+__all__ = [
+    "ModelInference",
+    "FileHandler",
+    "VisualizationHandler",
+    "EventHandlers",
+    "UIComponents",
+    "create_depth_visualization",
+    "save_to_gallery_func",
+    "get_scene_info",
+    "get_logo_base64",
+]

depth_anything_3/app/modules/event_handlers.py

@@ -0,0 +1,622 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Event handling module for Depth Anything 3 Gradio app.
+
+This module handles all event callbacks and user interactions.
+"""
+
+import os
+import time
+from glob import glob
+from typing import Any, Dict, List, Optional, Tuple
+import gradio as gr
+import numpy as np
+import torch
+
+from depth_anything_3.app.modules.file_handlers import FileHandler
+from depth_anything_3.app.modules.model_inference import ModelInference
+from depth_anything_3.utils.memory import cleanup_cuda_memory
+from depth_anything_3.app.modules.visualization import VisualizationHandler
+
+
+class EventHandlers:
+    """
+    Handles all event callbacks and user interactions for the Gradio app.
+    """
+
+    def __init__(self):
+        """Initialize the event handlers."""
+        self.model_inference = ModelInference()
+        self.file_handler = FileHandler()
+        self.visualization_handler = VisualizationHandler()
+
+    def clear_fields(self) -> None:
+        """
+        Clears the 3D viewer, the stored target_dir, and empties the gallery.
+        """
+        return None
+
+    def update_log(self) -> str:
+        """
+        Display a quick log message while waiting.
+        """
+        return "Loading and Reconstructing..."
+
+    def save_current_visualization(
+        self,
+        target_dir: str,
+        save_percentage: float,
+        show_cam: bool,
+        filter_black_bg: bool,
+        filter_white_bg: bool,
+        processed_data: Optional[Dict],
+        scene_name: str = "",
+    ) -> str:
+        """
+        Save current visualization results to gallery with specified save percentage.
+
+        Args:
+            target_dir: Directory containing results
+            save_percentage: Percentage of points to save (0-100)
+            show_cam: Whether to show cameras
+            filter_black_bg: Whether to filter black background
+            filter_white_bg: Whether to filter white background
+            processed_data: Processed data from reconstruction
+
+        Returns:
+            Status message
+        """
+        if not self.file_handler.is_valid_session_dir(target_dir):
+            return "No reconstruction available. Please run 'Reconstruct' first."
+
+        if processed_data is None:
+            return "No processed data available. Please run 'Reconstruct' first."
+
+        try:
+            # Add debug information
+            print("[DEBUG] save_current_visualization called with:")
+            print(f"  target_dir: {target_dir}")
+            print(f"  save_percentage: {save_percentage}")
+            print(f"  show_cam: {show_cam}")
+            print(f"  filter_black_bg: {filter_black_bg}")
+            print(f"  filter_white_bg: {filter_white_bg}")
+            print(f"  processed_data: {processed_data is not None}")
+
+            # Import the gallery save function
+            # Create gallery name with user input or auto-generated
+            import datetime
+
+            from .utils import save_to_gallery_func
+
+            timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+            if scene_name and scene_name.strip():
+                gallery_name = f"{scene_name.strip()}_{timestamp}_pct{save_percentage:.0f}"
+            else:
+                gallery_name = f"save_{timestamp}_pct{save_percentage:.0f}"
+
+            print(f"[DEBUG] Saving to gallery with name: {gallery_name}")
+
+            # Save entire process folder to gallery
+            success, message = save_to_gallery_func(
+                target_dir=target_dir, processed_data=processed_data, gallery_name=gallery_name
+            )
+
+            if success:
+                print(f"[DEBUG] Gallery save completed successfully: {message}")
+                return (
+                    "Successfully saved to gallery!\n"
+                    f"Gallery name: {gallery_name}\n"
+                    f"Save percentage: {save_percentage}%\n"
+                    f"Show cameras: {show_cam}\n"
+                    f"Filter black bg: {filter_black_bg}\n"
+                    f"Filter white bg: {filter_white_bg}\n\n"
+                    f"{message}"
+                )
+            else:
+                print(f"[DEBUG] Gallery save failed: {message}")
+                return f"Failed to save to gallery: {message}"
+
+        except Exception as e:
+            return f"Error saving visualization: {str(e)}"
+
+    def gradio_demo(
+        self,
+        target_dir: str,
+        show_cam: bool = True,
+        filter_black_bg: bool = False,
+        filter_white_bg: bool = False,
+        process_res_method: str = "upper_bound_resize",
+        save_percentage: float = 30.0,
+        num_max_points: int = 1_000_000,
+        infer_gs: bool = False,
+        ref_view_strategy: str = "saddle_balanced",
+        gs_trj_mode: str = "extend",
+        gs_video_quality: str = "high",
+    ) -> Tuple[
+        Optional[str],
+        str,
+        Optional[Dict],
+        Optional[np.ndarray],
+        Optional[np.ndarray],
+        str,
+        gr.Dropdown,
+        Optional[str],  # gs video path
+        gr.update,  # gs video visibility update
+        gr.update,  # gs info visibility update
+    ]:
+        """
+        Perform reconstruction using the already-created target_dir/images.
+
+        Args:
+            target_dir: Directory containing images
+            show_cam: Whether to show camera
+            filter_black_bg: Whether to filter black background
+            filter_white_bg: Whether to filter white background
+            process_res_method: Method for resizing input images
+            save_percentage: Filter percentage for point cloud
+            num_max_points: Maximum number of points
+            infer_gs: Whether to infer 3D Gaussian Splatting
+            ref_view_strategy: Reference view selection strategy
+
+        Returns:
+            Tuple of reconstruction results
+        """
+        if not self.file_handler.is_valid_session_dir(target_dir):
+            return (
+                None,
+                "No valid target directory found. Please upload first.",
+                None,
+                None,
+                None,
+                "",
+                None,
+                None,
+                gr.update(visible=False),  # gs_video
+                gr.update(visible=True),  # gs_info
+            )
+
+        start_time = time.time()
+        cleanup_cuda_memory()
+
+        # Get image files for logging
+        target_dir_images = os.path.join(target_dir, "images")
+        all_files = (
+            sorted(os.listdir(target_dir_images)) if os.path.isdir(target_dir_images) else []
+        )
+
+        print("Running DepthAnything3 model...")
+        print(f"Reference view strategy: {ref_view_strategy}")
+
+        with torch.no_grad():
+            prediction, processed_data = self.model_inference.run_inference(
+                target_dir,
+                process_res_method=process_res_method,
+                show_camera=show_cam,
+                save_percentage=save_percentage,
+                num_max_points=int(num_max_points * 1000),  # Convert K to actual count
+                infer_gs=infer_gs,
+                ref_view_strategy=ref_view_strategy,
+                gs_trj_mode=gs_trj_mode,
+                gs_video_quality=gs_video_quality,
+            )
+
+        # The GLB file is already generated by the API
+        glbfile = os.path.join(target_dir, "scene.glb")
+
+        # Handle 3DGS video based on infer_gs flag
+        gsvideo_path = None
+        gs_video_visible = False
+        gs_info_visible = True
+
+        if infer_gs:
+            try:
+                gsvideo_path = sorted(glob(os.path.join(target_dir, "gs_video", "*.mp4")))[-1]
+                gs_video_visible = True
+                gs_info_visible = False
+            except IndexError:
+                gsvideo_path = None
+                print("3DGS video not found, but infer_gs was enabled")
+
+        # Cleanup
+        cleanup_cuda_memory()
+
+        end_time = time.time()
+        print(f"Total time: {end_time - start_time:.2f} seconds")
+        log_msg = f"Reconstruction Success ({len(all_files)} frames). Waiting for visualization."
+
+        # Populate visualization tabs with processed data
+        depth_vis, measure_img, measure_depth_vis, measure_pts = (
+            self.visualization_handler.populate_visualization_tabs(processed_data)
+        )
+
+        # Update view selectors based on available views
+        depth_selector, measure_selector = self.visualization_handler.update_view_selectors(
+            processed_data
+        )
+
+        return (
+            glbfile,
+            log_msg,
+            processed_data,
+            measure_img,  # measure_image
+            measure_depth_vis,  # measure_depth_image
+            "",  # measure_text (empty initially)
+            measure_selector,  # measure_view_selector
+            gsvideo_path,
+            gr.update(visible=gs_video_visible),  # gs_video visibility
+            gr.update(visible=gs_info_visible),  # gs_info visibility
+        )
+
+    def update_visualization(
+        self,
+        target_dir: str,
+        show_cam: bool,
+        is_example: str,
+        filter_black_bg: bool = False,
+        filter_white_bg: bool = False,
+        process_res_method: str = "upper_bound_resize",
+    ) -> Tuple[gr.update, str]:
+        """
+        Reload saved predictions from npz, create (or reuse) the GLB for new parameters,
+        and return it for the 3D viewer.
+
+        Args:
+            target_dir: Directory containing results
+            show_cam: Whether to show camera
+            is_example: Whether this is an example scene
+            filter_black_bg: Whether to filter black background
+            filter_white_bg: Whether to filter white background
+            process_res_method: Method for resizing input images
+
+        Returns:
+            Tuple of (glb_file, log_message)
+        """
+        if not self.file_handler.is_valid_session_dir(target_dir):
+            return (
+                gr.update(),
+                "No reconstruction available. Please click the Reconstruct button first.",
+            )
+
+        # Check if GLB exists (could be cached example or reconstructed scene)
+        glbfile = os.path.join(target_dir, "scene.glb")
+        if os.path.exists(glbfile):
+            return (
+                glbfile,
+                (
+                    "Visualization loaded from cache."
+                    if is_example == "True"
+                    else "Visualization updated."
+                ),
+            )
+
+        # If no GLB but it's an example that hasn't been reconstructed yet
+        if is_example == "True":
+            return (
+                gr.update(),
+                "No reconstruction available. Please click the Reconstruct button first.",
+            )
+
+        # For non-examples, check predictions.npz
+        predictions_path = os.path.join(target_dir, "predictions.npz")
+        if not os.path.exists(predictions_path):
+            error_message = (
+                f"No reconstruction available at {predictions_path}. "
+                "Please run 'Reconstruct' first."
+            )
+            return gr.update(), error_message
+
+        try:
+            loaded = np.load(predictions_path, allow_pickle=False)
+            predictions = {key: loaded[key] for key in loaded.keys()}  # noqa: F841
+        except Exception as e:
+            return gr.update(), f"Cached results could not be read: {e}"
+
+        return (
+            glbfile,
+            "Visualization updated.",
+        )
+
+    def handle_uploads(
+        self,
+        input_video: Optional[str],
+        input_images: Optional[List],
+        s_time_interval: float = 10.0,
+    ) -> Tuple[Optional[str], Optional[str], Optional[List], Optional[str]]:
+        """
+        Handle file uploads and update gallery.
+
+        Args:
+            input_video: Path to input video file
+            input_images: List of input image files
+            s_time_interval: Sampling FPS (frames per second) for frame extraction
+
+        Returns:
+            Tuple of (reconstruction_output, target_dir, image_paths, log_message)
+        """
+        return self.file_handler.update_gallery_on_upload(
+            input_video, input_images, s_time_interval
+        )
+
+    def load_example_scene(self, scene_name: str, examples_dir: str = None) -> Tuple[
+        Optional[str],
+        Optional[str],
+        Optional[List],
+        str,
+        Optional[Dict],
+        gr.Dropdown,
+        Optional[str],
+        gr.update,
+        gr.update,
+    ]:
+        """
+        Load a scene from examples directory.
+
+        Args:
+            scene_name: Name of the scene to load
+            examples_dir: Path to examples directory (if None, uses workspace_dir/examples)
+
+        Returns:
+            Tuple of (reconstruction_output, target_dir, image_paths, log_message, processed_data, measure_view_selector, gs_video, gs_video_vis, gs_info_vis)  # noqa: E501
+        """
+        if examples_dir is None:
+            # Get workspace directory from environment variable
+            workspace_dir = os.environ.get("DA3_WORKSPACE_DIR", "gradio_workspace")
+            examples_dir = os.path.join(workspace_dir, "examples")
+
+        reconstruction_output, target_dir, image_paths, log_message = (
+            self.file_handler.load_example_scene(scene_name, examples_dir)
+        )
+
+        # Try to load cached processed data if available
+        processed_data = None
+        measure_view_selector = gr.Dropdown(choices=["View 1"], value="View 1")
+        gs_video_path = None
+        gs_video_visible = False
+        gs_info_visible = True
+
+        if self.file_handler.is_valid_session_dir(target_dir):
+            predictions_path = os.path.join(target_dir, "predictions.npz")
+            if os.path.exists(predictions_path):
+                try:
+                    # Load predictions from cache
+                    loaded = np.load(predictions_path, allow_pickle=False)
+                    predictions = {key: loaded[key] for key in loaded.keys()}
+
+                    # Reconstruct processed_data structure
+                    num_images = len(predictions.get("images", []))
+                    processed_data = {}
+
+                    for i in range(num_images):
+                        processed_data[i] = {
+                            "image": predictions["images"][i] if "images" in predictions else None,
+                            "depth": predictions["depths"][i] if "depths" in predictions else None,
+                            "depth_image": os.path.join(
+                                target_dir, "depth_vis", f"{i:04d}.jpg"  # Fixed: use .jpg not .png
+                            ),
+                            "intrinsics": (
+                                predictions["intrinsics"][i]
+                                if "intrinsics" in predictions
+                                and i < len(predictions["intrinsics"])
+                                else None
+                            ),
+                            "mask": None,
+                        }
+
+                    # Update measure view selector
+                    choices = [f"View {i + 1}" for i in range(num_images)]
+                    measure_view_selector = gr.Dropdown(choices=choices, value=choices[0])
+
+                except Exception as e:
+                    print(f"Error loading cached data: {e}")
+
+            # Check for cached 3DGS video
+            gs_video_dir = os.path.join(target_dir, "gs_video")
+            if os.path.exists(gs_video_dir):
+                try:
+                    from glob import glob
+
+                    gs_videos = sorted(glob(os.path.join(gs_video_dir, "*.mp4")))
+                    if gs_videos:
+                        gs_video_path = gs_videos[-1]
+                        gs_video_visible = True
+                        gs_info_visible = False
+                        print(f"Loaded cached 3DGS video: {gs_video_path}")
+                except Exception as e:
+                    print(f"Error loading cached 3DGS video: {e}")
+
+        return (
+            reconstruction_output,
+            target_dir,
+            image_paths,
+            log_message,
+            processed_data,
+            measure_view_selector,
+            gs_video_path,
+            gr.update(visible=gs_video_visible),
+            gr.update(visible=gs_info_visible),
+        )
+
+    def navigate_depth_view(
+        self,
+        processed_data: Optional[Dict[int, Dict[str, Any]]],
+        current_selector: str,
+        direction: int,
+    ) -> Tuple[str, Optional[str]]:
+        """
+        Navigate depth view.
+
+        Args:
+            processed_data: Processed data dictionary
+            current_selector: Current selector value
+            direction: Direction to navigate
+
+        Returns:
+            Tuple of (new_selector_value, depth_vis)
+        """
+        return self.visualization_handler.navigate_depth_view(
+            processed_data, current_selector, direction
+        )
+
+    def update_depth_view(
+        self, processed_data: Optional[Dict[int, Dict[str, Any]]], view_index: int
+    ) -> Optional[str]:
+        """
+        Update depth view for a specific view index.
+
+        Args:
+            processed_data: Processed data dictionary
+            view_index: Index of the view to update
+
+        Returns:
+            Path to depth visualization image or None
+        """
+        return self.visualization_handler.update_depth_view(processed_data, view_index)
+
+    def navigate_measure_view(
+        self,
+        processed_data: Optional[Dict[int, Dict[str, Any]]],
+        current_selector: str,
+        direction: int,
+    ) -> Tuple[str, Optional[np.ndarray], Optional[np.ndarray], List]:
+        """
+        Navigate measure view.
+
+        Args:
+            processed_data: Processed data dictionary
+            current_selector: Current selector value
+            direction: Direction to navigate
+
+        Returns:
+            Tuple of (new_selector_value, measure_image, depth_right_half, measure_points)
+        """
+        return self.visualization_handler.navigate_measure_view(
+            processed_data, current_selector, direction
+        )
+
+    def update_measure_view(
+        self, processed_data: Optional[Dict[int, Dict[str, Any]]], view_index: int
+    ) -> Tuple[Optional[np.ndarray], Optional[np.ndarray], List]:
+        """
+        Update measure view for a specific view index.
+
+        Args:
+            processed_data: Processed data dictionary
+            view_index: Index of the view to update
+
+        Returns:
+            Tuple of (measure_image, depth_right_half, measure_points)
+        """
+        return self.visualization_handler.update_measure_view(processed_data, view_index)
+
+    def measure(
+        self,
+        processed_data: Optional[Dict[int, Dict[str, Any]]],
+        measure_points: List,
+        current_view_selector: str,
+        event: gr.SelectData,
+    ) -> List:
+        """
+        Handle measurement on images.
+
+        Args:
+            processed_data: Processed data dictionary
+            measure_points: List of current measure points
+            current_view_selector: Current view selector value
+            event: Gradio select event
+
+        Returns:
+            List of [image, depth_right_half, measure_points, text]
+        """
+        return self.visualization_handler.measure(
+            processed_data, measure_points, current_view_selector, event
+        )
+
+    def select_first_frame(
+        self, image_gallery: List, selected_index: int = 0
+    ) -> Tuple[List, str, str]:
+        """
+        Select the first frame from the image gallery.
+
+        Args:
+            image_gallery: List of images in the gallery
+            selected_index: Index of the selected image (default: 0)
+
+        Returns:
+            Tuple of (updated_image_gallery, log_message, selected_frame_path)
+        """
+        try:
+            if not image_gallery or len(image_gallery) == 0:
+                return image_gallery, "No images available to select as first frame.", ""
+
+            # Handle None or invalid selected_index
+            if (
+                selected_index is None
+                or selected_index < 0
+                or selected_index >= len(image_gallery)
+            ):
+                selected_index = 0
+                print(f"Invalid selected_index: {selected_index}, using default: 0")
+
+            # Get the selected image based on index
+            selected_image = image_gallery[selected_index]
+            print(f"Selected image index: {selected_index}")
+            print(f"Total images: {len(image_gallery)}")
+
+            # Extract the file path from the selected image
+            selected_frame_path = ""
+            print(f"Selected image type: {type(selected_image)}")
+            print(f"Selected image: {selected_image}")
+
+            if isinstance(selected_image, tuple):
+                # Gradio Gallery returns tuple (path, None)
+                selected_frame_path = selected_image[0]
+            elif isinstance(selected_image, str):
+                selected_frame_path = selected_image
+            elif hasattr(selected_image, "name"):
+                selected_frame_path = selected_image.name
+            elif isinstance(selected_image, dict):
+                if "name" in selected_image:
+                    selected_frame_path = selected_image["name"]
+                elif "path" in selected_image:
+                    selected_frame_path = selected_image["path"]
+                elif "src" in selected_image:
+                    selected_frame_path = selected_image["src"]
+            else:
+                # Try to convert to string
+                selected_frame_path = str(selected_image)
+
+            print(f"Extracted path: {selected_frame_path}")
+
+            # Extract filename from the path for matching
+            import os
+
+            selected_filename = os.path.basename(selected_frame_path)
+            print(f"Selected filename: {selected_filename}")
+
+            # Move the selected image to the front
+            updated_gallery = [selected_image] + [
+                img for img in image_gallery if img != selected_image
+            ]
+
+            log_message = (
+                f"Selected frame: {selected_filename}. "
+                f"Moved to first position. Total frames: {len(updated_gallery)}"
+            )
+            return updated_gallery, log_message, selected_filename
+
+        except Exception as e:
+            print(f"Error selecting first frame: {e}")
+            return image_gallery, f"Error selecting first frame: {e}", ""

depth_anything_3/app/modules/file_handlers.py

@@ -0,0 +1,355 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+File handling module for Depth Anything 3 Gradio app.
+
+This module handles file uploads, video processing, and file operations.
+"""
+
+import os
+import shutil
+import time
+from datetime import datetime
+from typing import List, Optional, Tuple
+import cv2
+from PIL import Image
+from pillow_heif import register_heif_opener
+
+register_heif_opener()
+
+
+ALLOWED_IMAGE_EXTENSIONS = {
+    ".png",
+    ".jpg",
+    ".jpeg",
+    ".bmp",
+    ".tiff",
+    ".tif",
+    ".webp",
+    ".heic",
+    ".heif",
+}
+
+
+def _looks_like_image(file_path: str) -> bool:
+    """Best-effort check that a file is actually a decodable image, not just named like one."""
+    try:
+        with Image.open(file_path) as img:
+            img.verify()
+        return True
+    except Exception:
+        return False
+
+
+class FileHandler:
+    """
+    Handles file uploads and processing for the Gradio app.
+    """
+
+    def __init__(self):
+        """Initialize the file handler."""
+
+    def is_valid_session_dir(self, target_dir: Optional[str]) -> bool:
+        """Check that target_dir is a session/example directory this handler itself
+        creates under the configured workspace, rather than an arbitrary path."""
+        if not target_dir or target_dir == "None":
+            return False
+
+        try:
+            workspace_dir = os.environ.get("DA3_WORKSPACE_DIR", "gradio_workspace")
+            input_images_dir = os.path.realpath(os.path.join(workspace_dir, "input_images"))
+            real_target = os.path.realpath(target_dir)
+
+            if real_target != input_images_dir and not real_target.startswith(
+                input_images_dir + os.sep
+            ):
+                return False
+
+            rel = os.path.relpath(real_target, input_images_dir)
+            if os.sep in rel or not (rel.startswith("session_") or rel.startswith("example_")):
+                return False
+
+            return os.path.isdir(real_target)
+        except (OSError, ValueError):
+            return False
+
+    def handle_uploads(
+        self,
+        input_video: Optional[str],
+        input_images: Optional[List],
+        s_time_interval: float = 10.0,
+    ) -> Tuple[str, List[str]]:
+        """
+        Create a new 'target_dir' + 'images' subfolder, and place user-uploaded
+        images or extracted frames from video into it.
+
+        Args:
+            input_video: Path to input video file
+            input_images: List of input image files
+            s_time_interval: Sampling FPS (frames per second) for frame extraction
+
+        Returns:
+            Tuple of (target_dir, image_paths)
+        """
+        start_time = time.time()
+
+        # Get workspace directory from environment variable or use default
+        workspace_dir = os.environ.get("DA3_WORKSPACE_DIR", "gradio_workspace")
+        if not os.path.exists(workspace_dir):
+            os.makedirs(workspace_dir)
+
+        # Create input_images subdirectory
+        input_images_dir = os.path.join(workspace_dir, "input_images")
+        if not os.path.exists(input_images_dir):
+            os.makedirs(input_images_dir)
+
+        # Create a unique folder name within input_images
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
+        target_dir = os.path.join(input_images_dir, f"session_{timestamp}")
+        target_dir_images = os.path.join(target_dir, "images")
+
+        # Clean up if somehow that folder already exists
+        if os.path.exists(target_dir):
+            shutil.rmtree(target_dir)
+        os.makedirs(target_dir)
+        os.makedirs(target_dir_images)
+
+        image_paths = []
+
+        # Handle images
+        if input_images is not None:
+            image_paths.extend(self._process_images(input_images, target_dir_images))
+
+        # Handle video
+        if input_video is not None:
+            image_paths.extend(
+                self._process_video(input_video, target_dir_images, s_time_interval)
+            )
+
+        # Sort final images for gallery
+        image_paths = sorted(image_paths)
+
+        end_time = time.time()
+        print(f"Files copied to {target_dir_images}; took {end_time - start_time:.3f} seconds")
+        return target_dir, image_paths
+
+    def _process_images(self, input_images: List, target_dir_images: str) -> List[str]:
+        """
+        Process uploaded images.
+
+        Args:
+            input_images: List of input image files
+            target_dir_images: Target directory for images
+
+        Returns:
+            List of processed image paths
+        """
+        image_paths = []
+
+        for file_data in input_images:
+            if isinstance(file_data, dict) and "name" in file_data:
+                file_path = file_data["name"]
+            else:
+                file_path = file_data
+
+            file_ext = os.path.splitext(file_path)[1].lower()
+            if file_ext not in ALLOWED_IMAGE_EXTENSIONS or not _looks_like_image(file_path):
+                print(f"Skipping non-image upload: {os.path.basename(file_path)}")
+                continue
+
+            # Check if the file is a HEIC image
+            if file_ext in [".heic", ".heif"]:
+                # Convert HEIC to JPEG for better gallery compatibility
+                try:
+                    with Image.open(file_path) as img:
+                        # Convert to RGB if necessary (HEIC can have different color modes)
+                        if img.mode not in ("RGB", "L"):
+                            img = img.convert("RGB")
+
+                        # Create JPEG filename
+                        base_name = os.path.splitext(os.path.basename(file_path))[0]
+                        dst_path = os.path.join(target_dir_images, f"{base_name}.jpg")
+
+                        # Save as JPEG with high quality
+                        img.save(dst_path, "JPEG", quality=95)
+                        image_paths.append(dst_path)
+                        print(
+                            f"Converted HEIC to JPEG: {os.path.basename(file_path)} -> "
+                            f"{os.path.basename(dst_path)}"
+                        )
+                except Exception as e:
+                    print(f"Error converting HEIC file {file_path}: {e}")
+                    # Fall back to copying as is
+                    dst_path = os.path.join(target_dir_images, os.path.basename(file_path))
+                    shutil.copy(file_path, dst_path)
+                    image_paths.append(dst_path)
+            else:
+                # Regular image files - copy as is
+                dst_path = os.path.join(target_dir_images, os.path.basename(file_path))
+                shutil.copy(file_path, dst_path)
+                image_paths.append(dst_path)
+
+        return image_paths
+
+    def _process_video(
+        self, input_video: str, target_dir_images: str, s_time_interval: float
+    ) -> List[str]:
+        """
+        Process video file and extract frames.
+
+        Args:
+            input_video: Path to input video file
+            target_dir_images: Target directory for extracted frames
+            s_time_interval: Sampling FPS (frames per second) for frame extraction
+
+        Returns:
+            List of extracted frame paths
+        """
+        image_paths = []
+
+        if isinstance(input_video, dict) and "name" in input_video:
+            video_path = input_video["name"]
+        else:
+            video_path = input_video
+
+        vs = cv2.VideoCapture(video_path)
+        fps = vs.get(cv2.CAP_PROP_FPS)
+        frame_interval = max(1, int(fps / s_time_interval))  # Convert FPS to frame interval
+
+        count = 0
+        video_frame_num = 0
+        while True:
+            gotit, frame = vs.read()
+            if not gotit:
+                break
+            count += 1
+            if count % frame_interval == 0:
+                image_path = os.path.join(target_dir_images, f"{video_frame_num:06}.png")
+                cv2.imwrite(image_path, frame)
+                image_paths.append(image_path)
+                video_frame_num += 1
+
+        return image_paths
+
+    def update_gallery_on_upload(
+        self,
+        input_video: Optional[str],
+        input_images: Optional[List],
+        s_time_interval: float = 10.0,
+    ) -> Tuple[Optional[str], Optional[str], Optional[List], Optional[str]]:
+        """
+        Handle file uploads and update gallery.
+
+        Args:
+            input_video: Path to input video file
+            input_images: List of input image files
+            s_time_interval: Sampling FPS (frames per second) for frame extraction
+
+        Returns:
+            Tuple of (reconstruction_output, target_dir, image_paths, log_message)
+        """
+        if not input_video and not input_images:
+            return None, None, None, None
+
+        target_dir, image_paths = self.handle_uploads(input_video, input_images, s_time_interval)
+        return (
+            None,
+            target_dir,
+            image_paths,
+            "Upload complete. Click 'Reconstruct' to begin 3D processing.",
+        )
+
+    def load_example_scene(
+        self, scene_name: str, examples_dir: str = "examples"
+    ) -> Tuple[Optional[str], Optional[str], Optional[List], str]:
+        """
+        Load a scene from examples directory.
+
+        Args:
+            scene_name: Name of the scene to load
+            examples_dir: Path to examples directory
+
+        Returns:
+            Tuple of (reconstruction_output, target_dir, image_paths, log_message)
+        """
+        from depth_anything_3.app.modules.utils import get_scene_info
+
+        scenes = get_scene_info(examples_dir)
+
+        # Find the selected scene
+        selected_scene = None
+        for scene in scenes:
+            if scene["name"] == scene_name:
+                selected_scene = scene
+                break
+
+        if selected_scene is None:
+            return None, None, None, "Scene not found"
+
+        # Use fixed directory name for examples (not timestamp-based)
+        workspace_dir = os.environ.get("DA3_WORKSPACE_DIR", "gradio_workspace")
+        input_images_dir = os.path.join(workspace_dir, "input_images")
+        if not os.path.exists(input_images_dir):
+            os.makedirs(input_images_dir)
+
+        # Create a fixed folder name based on scene name
+        target_dir = os.path.join(input_images_dir, f"example_{scene_name}")
+        target_dir_images = os.path.join(target_dir, "images")
+
+        # Check if already cached (GLB file exists)
+        glb_path = os.path.join(target_dir, "scene.glb")
+        is_cached = os.path.exists(glb_path)
+
+        # Create directory if it doesn't exist
+        if not os.path.exists(target_dir):
+            os.makedirs(target_dir)
+            os.makedirs(target_dir_images)
+
+        # Copy images if directory is new or empty
+        if not os.path.exists(target_dir_images) or len(os.listdir(target_dir_images)) == 0:
+            os.makedirs(target_dir_images, exist_ok=True)
+            image_paths = []
+            for file_path in selected_scene["image_files"]:
+                dst_path = os.path.join(target_dir_images, os.path.basename(file_path))
+                shutil.copy(file_path, dst_path)
+                image_paths.append(dst_path)
+        else:
+            # Use existing images
+            image_paths = sorted(
+                [
+                    os.path.join(target_dir_images, f)
+                    for f in os.listdir(target_dir_images)
+                    if f.lower().endswith((".png", ".jpg", ".jpeg", ".bmp", ".tiff", ".tif"))
+                ]
+            )
+
+        # Return cached GLB if available
+        if is_cached:
+            return (
+                glb_path,  # Return cached reconstruction
+                target_dir,  # Set target directory
+                image_paths,  # Set gallery
+                f"Loaded cached scene '{scene_name}' with {selected_scene['num_images']} images.",
+            )
+        else:
+            return (
+                None,  # No cached reconstruction
+                target_dir,  # Set target directory
+                image_paths,  # Set gallery
+                (
+                    f"Loaded scene '{scene_name}' with {selected_scene['num_images']} images. "
+                    "Click 'Reconstruct' to begin 3D processing."
+                ),
+            )

depth_anything_3/app/modules/model_inference.py

@@ -0,0 +1,260 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Model inference module for Depth Anything 3 Gradio app.
+
+This module handles all model-related operations including inference,
+data processing, and result preparation.
+"""
+
+import glob
+import os
+from typing import Any, Dict, Optional, Tuple
+import numpy as np
+import torch
+
+from depth_anything_3.api import DepthAnything3
+from depth_anything_3.utils.memory import cleanup_cuda_memory
+from depth_anything_3.utils.export.glb import export_to_glb
+from depth_anything_3.utils.export.gs import export_to_gs_video
+
+
+class ModelInference:
+    """
+    Handles model inference and data processing for Depth Anything 3.
+    """
+
+    def __init__(self):
+        """Initialize the model inference handler."""
+        self.model = None
+
+    def initialize_model(self, device: str = "cuda") -> None:
+        """
+        Initialize the DepthAnything3 model.
+
+        Args:
+            device: Device to load the model on
+        """
+        if self.model is None:
+            # Get model directory from environment variable or use default
+            model_dir = os.environ.get(
+                "DA3_MODEL_DIR", "/dev/shm/da3_models/DA3HF-VITG-METRIC_VITL"
+            )
+            self.model = DepthAnything3.from_pretrained(model_dir)
+            self.model = self.model.to(device)
+        else:
+            self.model = self.model.to(device)
+
+        self.model.eval()
+
+    def run_inference(
+        self,
+        target_dir: str,
+        filter_black_bg: bool = False,
+        filter_white_bg: bool = False,
+        process_res_method: str = "upper_bound_resize",
+        show_camera: bool = True,
+        save_percentage: float = 30.0,
+        num_max_points: int = 1_000_000,
+        infer_gs: bool = False,
+        ref_view_strategy: str = "saddle_balanced",
+        gs_trj_mode: str = "extend",
+        gs_video_quality: str = "high",
+    ) -> Tuple[Any, Dict[int, Dict[str, Any]]]:
+        """
+        Run DepthAnything3 model inference on images.
+
+        Args:
+            target_dir: Directory containing images
+            filter_black_bg: Whether to filter black background
+            filter_white_bg: Whether to filter white background
+            process_res_method: Method for resizing input images
+            show_camera: Whether to show camera in 3D view
+            save_percentage: Percentage of points to save (0-100)
+            num_max_points: Maximum number of points in point cloud
+            infer_gs: Whether to infer 3D Gaussian Splatting
+            ref_view_strategy: Reference view selection strategy
+            gs_trj_mode: Trajectory mode for 3DGS
+            gs_video_quality: Video quality for 3DGS
+
+        Returns:
+            Tuple of (prediction, processed_data)
+        """
+        print(f"Processing images from {target_dir}")
+
+        # Device check
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        device = torch.device(device)
+
+        # Initialize model if needed
+        self.initialize_model(device)
+
+        # Get image paths
+        print("Loading images...")
+        image_folder_path = os.path.join(target_dir, "images")
+        all_image_paths = sorted(glob.glob(os.path.join(image_folder_path, "*")))
+
+        # Filter for image files
+        image_extensions = [".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".tif"]
+        all_image_paths = [
+            path
+            for path in all_image_paths
+            if any(path.lower().endswith(ext) for ext in image_extensions)
+        ]
+
+        print(f"Found {len(all_image_paths)} images")
+        print(f"All image paths: {all_image_paths}")
+
+        # Use sorted image order (reference view will be selected automatically)
+        image_paths = all_image_paths
+        print(f"Reference view selection strategy: {ref_view_strategy}")
+
+        if len(image_paths) == 0:
+            raise ValueError("No images found. Check your upload.")
+
+        # Map UI options to actual method names
+        method_mapping = {"high_res": "lower_bound_resize", "low_res": "upper_bound_resize"}
+        actual_method = method_mapping.get(process_res_method, "upper_bound_crop")
+
+        # Run model inference
+        print(f"Running inference with method: {actual_method}")
+        with torch.no_grad():
+            prediction = self.model.inference(
+                image_paths,
+                export_dir=None,
+                process_res_method=actual_method,
+                infer_gs=infer_gs,
+                ref_view_strategy=ref_view_strategy,
+            )
+        # num_max_points: int = 1_000_000,
+        export_to_glb(
+            prediction,
+            filter_black_bg=filter_black_bg,
+            filter_white_bg=filter_white_bg,
+            export_dir=target_dir,
+            show_cameras=show_camera,
+            conf_thresh_percentile=save_percentage,
+            num_max_points=int(num_max_points),
+        )
+
+        # export to gs video if needed
+        if infer_gs:
+            mode_mapping = {"extend": "extend", "smooth": "interpolate_smooth"}
+            print(f"GS mode: {gs_trj_mode}; Backend mode: {mode_mapping[gs_trj_mode]}")
+            export_to_gs_video(
+                prediction,
+                export_dir=target_dir,
+                chunk_size=4,
+                trj_mode=mode_mapping.get(gs_trj_mode, "extend"),
+                enable_tqdm=True,
+                vis_depth="hcat",
+                video_quality=gs_video_quality,
+            )
+
+        # Save predictions.npz for caching metric depth data
+        self._save_predictions_cache(target_dir, prediction)
+
+        # Process results
+        processed_data = self._process_results(target_dir, prediction, image_paths)
+
+        # Clean up using centralized memory utilities for consistency with backend
+        cleanup_cuda_memory()
+
+        return prediction, processed_data
+
+    def _save_predictions_cache(self, target_dir: str, prediction: Any) -> None:
+        """
+        Save predictions data to predictions.npz for caching.
+
+        Args:
+            target_dir: Directory to save the cache
+            prediction: Model prediction object
+        """
+        try:
+            output_file = os.path.join(target_dir, "predictions.npz")
+
+            # Build save dict with prediction data
+            save_dict = {}
+
+            # Save processed images if available
+            if prediction.processed_images is not None:
+                save_dict["images"] = prediction.processed_images
+
+            # Save depth data
+            if prediction.depth is not None:
+                save_dict["depths"] = np.round(prediction.depth, 6)
+
+            # Save confidence if available
+            if prediction.conf is not None:
+                save_dict["conf"] = np.round(prediction.conf, 2)
+
+            # Save camera parameters
+            if prediction.extrinsics is not None:
+                save_dict["extrinsics"] = prediction.extrinsics
+            if prediction.intrinsics is not None:
+                save_dict["intrinsics"] = prediction.intrinsics
+
+            # Save to file
+            np.savez_compressed(output_file, **save_dict)
+            print(f"Saved predictions cache to: {output_file}")
+
+        except Exception as e:
+            print(f"Warning: Failed to save predictions cache: {e}")
+
+    def _process_results(
+        self, target_dir: str, prediction: Any, image_paths: list
+    ) -> Dict[int, Dict[str, Any]]:
+        """
+        Process model results into structured data.
+
+        Args:
+            target_dir: Directory containing results
+            prediction: Model prediction object
+            image_paths: List of input image paths
+
+        Returns:
+            Dictionary containing processed data for each view
+        """
+        processed_data = {}
+
+        # Read generated depth visualization files
+        depth_vis_dir = os.path.join(target_dir, "depth_vis")
+
+        if os.path.exists(depth_vis_dir):
+            depth_files = sorted(glob.glob(os.path.join(depth_vis_dir, "*.jpg")))
+            for i, depth_file in enumerate(depth_files):
+                # Use processed images directly from API
+                processed_image = None
+                if prediction.processed_images is not None and i < len(
+                    prediction.processed_images
+                ):
+                    processed_image = prediction.processed_images[i]
+
+                processed_data[i] = {
+                    "depth_image": depth_file,
+                    "image": processed_image,
+                    "original_image_path": image_paths[i] if i < len(image_paths) else None,
+                    "depth": prediction.depth[i] if i < len(prediction.depth) else None,
+                    "intrinsics": (
+                        prediction.intrinsics[i]
+                        if prediction.intrinsics is not None and i < len(prediction.intrinsics)
+                        else None
+                    ),
+                    "mask": None,  # No mask information available
+                }
+
+        return processed_data
+
+    # cleanup() removed: call cleanup_cuda_memory() directly where needed.

depth_anything_3/app/modules/ui_components.py

@@ -0,0 +1,477 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+UI components module for Depth Anything 3 Gradio app.
+
+This module contains UI component definitions and layout functions.
+"""
+
+import os
+from typing import Any, Dict, List, Tuple
+import gradio as gr
+
+from depth_anything_3.app.modules.utils import get_logo_base64, get_scene_info
+
+
+class UIComponents:
+    """
+    Handles UI component creation and layout for the Gradio app.
+    """
+
+    def __init__(self):
+        """Initialize the UI components handler."""
+
+    def create_upload_section(self) -> Tuple[gr.Video, gr.Slider, gr.File, gr.Gallery]:
+        """
+        Create the upload section with video, images, and gallery components.
+
+        Returns:
+            A tuple of Gradio components: (input_video, s_time_interval, input_images, image_gallery).
+        """
+        input_video = gr.Video(label="Upload Video", interactive=True)
+        s_time_interval = gr.Slider(
+            minimum=0.1,
+            maximum=60,
+            value=10,
+            step=0.1,
+            label="Sampling FPS (Frames Per Second)",
+            interactive=True,
+            visible=True,
+        )
+        input_images = gr.File(file_count="multiple", label="Upload Images", interactive=True)
+        image_gallery = gr.Gallery(
+            label="Preview",
+            columns=4,
+            height="300px",
+            show_download_button=True,
+            object_fit="contain",
+            preview=True,
+            interactive=False,
+        )
+
+        return input_video, s_time_interval, input_images, image_gallery
+
+    def create_3d_viewer_section(self) -> gr.Model3D:
+        """
+        Create the 3D viewer component.
+
+        Returns:
+            3D model viewer component
+        """
+        return gr.Model3D(
+            height=520,
+            zoom_speed=0.5,
+            pan_speed=0.5,
+            clear_color=[0.0, 0.0, 0.0, 0.0],
+            key="persistent_3d_viewer",
+            elem_id="reconstruction_3d_viewer",
+        )
+
+    def create_nvs_video(self) -> Tuple[gr.Video, gr.Markdown]:
+        """
+        Create the 3DGS rendered video display component and info message.
+
+        Returns:
+            Tuple of (video component, info message component)
+        """
+        with gr.Column():
+            gs_info = gr.Markdown(
+                (
+                    "‼️ **3D Gaussian Splatting rendering is currently DISABLED.** <br><br><br>"
+                    "To render novel views from 3DGS, "
+                    "enable **Infer 3D Gaussian Splatting** below. <br>"
+                    "Next, in **Visualization Options**, "
+                    "*optionally* configure the **rendering trajectory** (default: smooth) "
+                    "and **video quality** (default: low), "
+                    "then click **Reconstruct**."
+                ),
+                visible=True,
+                height=520,
+            )
+            gs_video = gr.Video(
+                height=520,
+                label="3DGS Rendered NVS Video (depth shown for reference only)",
+                interactive=False,
+                visible=False,
+            )
+        return gs_video, gs_info
+
+    def create_depth_section(self) -> Tuple[gr.Button, gr.Dropdown, gr.Button, gr.Image]:
+        """
+        Create the depth visualization section.
+
+        Returns:
+            A tuple of (prev_depth_btn, depth_view_selector, next_depth_btn, depth_map)
+        """
+        with gr.Row(elem_classes=["navigation-row"]):
+            prev_depth_btn = gr.Button("◀ Previous", size="sm", scale=1)
+            depth_view_selector = gr.Dropdown(
+                choices=["View 1"],
+                value="View 1",
+                label="Select View",
+                scale=2,
+                interactive=True,
+                allow_custom_value=True,
+            )
+            next_depth_btn = gr.Button("Next ▶", size="sm", scale=1)
+        depth_map = gr.Image(
+            type="numpy",
+            label="Colorized Depth Map",
+            format="png",
+            interactive=False,
+        )
+
+        return prev_depth_btn, depth_view_selector, next_depth_btn, depth_map
+
+    def create_measure_section(
+        self,
+    ) -> Tuple[gr.Button, gr.Dropdown, gr.Button, gr.Image, gr.Image, gr.Markdown]:
+        """
+        Create the measurement section.
+
+        Returns:
+            A tuple of (prev_measure_btn, measure_view_selector, next_measure_btn, measure_image,
+            measure_depth_image, measure_text)
+        """
+        from depth_anything_3.app.css_and_html import MEASURE_INSTRUCTIONS_HTML
+
+        gr.Markdown(MEASURE_INSTRUCTIONS_HTML)
+        with gr.Row(elem_classes=["navigation-row"]):
+            prev_measure_btn = gr.Button("◀ Previous", size="sm", scale=1)
+            measure_view_selector = gr.Dropdown(
+                choices=["View 1"],
+                value="View 1",
+                label="Select View",
+                scale=2,
+                interactive=True,
+                allow_custom_value=True,
+            )
+            next_measure_btn = gr.Button("Next ▶", size="sm", scale=1)
+        with gr.Row():
+            measure_image = gr.Image(
+                type="numpy",
+                show_label=False,
+                format="webp",
+                interactive=False,
+                sources=[],
+                label="RGB Image",
+                scale=1,
+                height=275,
+            )
+            measure_depth_image = gr.Image(
+                type="numpy",
+                show_label=False,
+                format="webp",
+                interactive=False,
+                sources=[],
+                label="Depth Visualization (Right Half)",
+                scale=1,
+                height=275,
+            )
+        gr.Markdown(
+            "**Note:** Images have been adjusted to model processing size. "
+            "Click two points on the RGB image to measure distance."
+        )
+        measure_text = gr.Markdown("")
+
+        return (
+            prev_measure_btn,
+            measure_view_selector,
+            next_measure_btn,
+            measure_image,
+            measure_depth_image,
+            measure_text,
+        )
+
+    def create_inference_control_section(self) -> Tuple[gr.Dropdown, gr.Checkbox, gr.Dropdown]:
+        """
+        Create the inference control section (before inference).
+
+        Returns:
+            Tuple of (process_res_method_dropdown, infer_gs, ref_view_strategy)
+        """
+        with gr.Row():
+            process_res_method_dropdown = gr.Dropdown(
+                choices=["high_res", "low_res"],
+                value="low_res",
+                label="Image Processing Method",
+                info="low_res for much more images",
+                scale=1,
+            )
+            # Modify line 220, add color class
+            infer_gs = gr.Checkbox(
+                label="Infer 3D Gaussian Splatting",
+                value=False,
+                info=(
+                    'Enable novel view rendering from 3DGS (<i class="fas fa-triangle-exclamation '
+                    'fa-color-red"></i> requires extra processing time)'
+                ),
+                scale=1,
+            )
+            ref_view_strategy = gr.Dropdown(
+                choices=["saddle_balanced", "saddle_sim_range", "first", "middle"],
+                value="saddle_balanced",
+                label="Reference View Strategy",
+                info="Strategy for selecting reference view from multiple inputs",
+                scale=1,
+            )
+
+        return (process_res_method_dropdown, infer_gs, ref_view_strategy)
+
+    def create_display_control_section(
+        self,
+    ) -> Tuple[
+        gr.Checkbox,
+        gr.Checkbox,
+        gr.Checkbox,
+        gr.Slider,
+        gr.Slider,
+        gr.Dropdown,
+        gr.Dropdown,
+        gr.Button,
+        gr.ClearButton,
+    ]:
+        """
+        Create the display control section (options for visualization).
+
+        Returns:
+            Tuple of display control components including buttons
+        """
+        with gr.Column():
+            # 3DGS options at the top
+            with gr.Row():
+                gs_trj_mode = gr.Dropdown(
+                    choices=["smooth", "extend"],
+                    value="smooth",
+                    label=("Rendering trajectory for 3DGS viewpoints (requires n_views ≥ 2)"),
+                    info=("'smooth' for view interpolation; 'extend' for longer trajectory"),
+                    visible=False,  # initially hidden
+                )
+                gs_video_quality = gr.Dropdown(
+                    choices=["low", "medium", "high"],
+                    value="low",
+                    label=("Video quality for 3DGS rendered outputs"),
+                    info=("'low' for faster loading speed; 'high' for better visual quality"),
+                    visible=False,  # initially hidden
+                )
+
+            # Reconstruct and Clear buttons (before Visualization Options)
+            with gr.Row():
+                submit_btn = gr.Button("Reconstruct", scale=1, variant="primary")
+                clear_btn = gr.ClearButton(scale=1)
+
+            gr.Markdown("### Visualization Options: (Click Reconstruct to update)")
+            show_cam = gr.Checkbox(label="Show Camera", value=True)
+            filter_black_bg = gr.Checkbox(label="Filter Black Background", value=False)
+            filter_white_bg = gr.Checkbox(label="Filter White Background", value=False)
+            save_percentage = gr.Slider(
+                minimum=0,
+                maximum=100,
+                value=10,
+                step=1,
+                label="Filter Percentage",
+                info="Confidence Threshold (%): Higher values filter more points.",
+            )
+            num_max_points = gr.Slider(
+                minimum=1000,
+                maximum=100000,
+                value=1000,
+                step=1000,
+                label="Max Points (K points)",
+                info="Maximum number of points to export to GLB (in thousands)",
+            )
+
+        return (
+            show_cam,
+            filter_black_bg,
+            filter_white_bg,
+            save_percentage,
+            num_max_points,
+            gs_trj_mode,
+            gs_video_quality,
+            submit_btn,
+            clear_btn,
+        )
+
+    def create_control_section(
+        self,
+    ) -> Tuple[
+        gr.Button,
+        gr.ClearButton,
+        gr.Dropdown,
+        gr.Checkbox,
+        gr.Checkbox,
+        gr.Checkbox,
+        gr.Checkbox,
+        gr.Checkbox,
+        gr.Dropdown,
+        gr.Checkbox,
+        gr.Textbox,
+    ]:
+        """
+        Create the control section with buttons and options.
+
+        Returns:
+            Tuple of control components
+        """
+        with gr.Row():
+            submit_btn = gr.Button("Reconstruct", scale=1, variant="primary")
+            clear_btn = gr.ClearButton(
+                scale=1,
+            )
+
+        with gr.Row():
+            frame_filter = gr.Dropdown(
+                choices=["All"], value="All", label="Show Points from Frame"
+            )
+            with gr.Column():
+                gr.Markdown("### Visualization Option: (Click Reconstruct to update)")
+                show_cam = gr.Checkbox(label="Show Camera", value=True)
+                show_mesh = gr.Checkbox(label="Show Mesh", value=True)
+                filter_black_bg = gr.Checkbox(label="Filter Black Background", value=False)
+                filter_white_bg = gr.Checkbox(label="Filter White Background", value=False)
+                gr.Markdown("### Reconstruction Options: (updated on next run)")
+                apply_mask_checkbox = gr.Checkbox(
+                    label="Apply mask for predicted ambiguous depth classes & edges",
+                    value=True,
+                )
+                process_res_method_dropdown = gr.Dropdown(
+                    choices=[
+                        "upper_bound_resize",
+                        "upper_bound_crop",
+                        "lower_bound_resize",
+                        "lower_bound_crop",
+                    ],
+                    value="upper_bound_resize",
+                    label="Image Processing Method",
+                    info="Method for resizing input images",
+                )
+                save_to_gallery_checkbox = gr.Checkbox(
+                    label="Save to Gallery",
+                    value=False,
+                    info="Save current reconstruction results to gallery directory",
+                )
+                gallery_name_input = gr.Textbox(
+                    label="Gallery Name",
+                    placeholder="Enter a name for the gallery folder",
+                    value="",
+                    info="Leave empty for auto-generated name with timestamp",
+                )
+
+        return (
+            submit_btn,
+            clear_btn,
+            frame_filter,
+            show_cam,
+            show_mesh,
+            filter_black_bg,
+            filter_white_bg,
+            apply_mask_checkbox,
+            process_res_method_dropdown,
+            save_to_gallery_checkbox,
+            gallery_name_input,
+        )
+
+    def create_example_scenes_section(self) -> List[Dict[str, Any]]:
+        """
+        Create the example scenes section.
+
+        Returns:
+            List of scene information dictionaries
+        """
+        # Get workspace directory from environment variable
+        workspace_dir = os.environ.get("DA3_WORKSPACE_DIR", "gradio_workspace")
+        examples_dir = os.path.join(workspace_dir, "examples")
+
+        # Get scene information
+        scenes = get_scene_info(examples_dir)
+
+        return scenes
+
+    def create_example_scene_grid(self, scenes: List[Dict[str, Any]]) -> List[gr.Image]:
+        """
+        Create the example scene grid.
+
+        Args:
+            scenes: List of scene information dictionaries
+
+        Returns:
+            List of scene image components
+        """
+        scene_components = []
+
+        if scenes:
+            for i in range(0, len(scenes), 4):  # Process 4 scenes per row
+                with gr.Row():
+                    for j in range(4):
+                        scene_idx = i + j
+                        if scene_idx < len(scenes):
+                            scene = scenes[scene_idx]
+                            with gr.Column(scale=1, elem_classes=["clickable-thumbnail"]):
+                                # Clickable thumbnail
+                                scene_img = gr.Image(
+                                    value=scene["thumbnail"],
+                                    height=150,
+                                    interactive=False,
+                                    show_label=False,
+                                    elem_id=f"scene_thumb_{scene['name']}",
+                                    sources=[],
+                                )
+                                scene_components.append(scene_img)
+
+                                # Scene name and image count as text below thumbnail
+                                gr.Markdown(
+                                    f"**{scene['name']}** \n {scene['num_images']} images",
+                                    elem_classes=["scene-info"],
+                                )
+                        else:
+                            # Empty column to maintain grid structure
+                            with gr.Column(scale=1):
+                                pass
+
+        return scene_components
+
+    def create_header_section(self) -> gr.HTML:
+        """
+        Create the header section with logo and title.
+
+        Returns:
+            Header HTML component
+        """
+        from depth_anything_3.app.css_and_html import get_header_html
+
+        return gr.HTML(get_header_html(get_logo_base64()))
+
+    def create_description_section(self) -> gr.HTML:
+        """
+        Create the description section.
+
+        Returns:
+            Description HTML component
+        """
+        from depth_anything_3.app.css_and_html import get_description_html
+
+        return gr.HTML(get_description_html())
+
+    def create_acknowledgements_section(self) -> gr.HTML:
+        """
+        Create the acknowledgements section.
+
+        Returns:
+            Acknowledgements HTML component
+        """
+        from depth_anything_3.app.css_and_html import get_acknowledgements_html
+
+        return gr.HTML(get_acknowledgements_html())

depth_anything_3/app/modules/utils.py

@@ -0,0 +1,207 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Utility functions for Depth Anything 3 Gradio app.
+
+This module contains helper functions for data processing, visualization,
+and file operations.
+"""
+
+
+import json
+import os
+import shutil
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Tuple
+import numpy as np
+
+def create_depth_visualization(depth: np.ndarray) -> Optional[np.ndarray]:
+    """
+    Create a colored depth visualization.
+
+    Args:
+        depth: Depth array
+
+    Returns:
+        Colored depth visualization or None
+    """
+    if depth is None:
+        return None
+
+    # Normalize depth to 0-1 range
+    depth_min = depth[depth > 0].min() if (depth > 0).any() else 0
+    depth_max = depth.max()
+
+    if depth_max <= depth_min:
+        return None
+
+    # Normalize depth
+    depth_norm = (depth - depth_min) / (depth_max - depth_min)
+    depth_norm = np.clip(depth_norm, 0, 1)
+
+    # Apply colormap (using matplotlib's viridis colormap)
+    import matplotlib.cm as cm
+
+    # Convert to colored image
+    depth_colored = cm.viridis(depth_norm)[:, :, :3]  # Remove alpha channel
+    depth_colored = (depth_colored * 255).astype(np.uint8)
+
+    return depth_colored
+
+
+def save_to_gallery_func(
+    target_dir: str, processed_data: Dict[int, Dict[str, Any]], gallery_name: Optional[str] = None
+) -> Tuple[bool, str]:
+    """
+    Save the current reconstruction results to the gallery directory.
+
+    Args:
+        target_dir: Source directory containing reconstruction results
+        processed_data: Processed data dictionary
+        gallery_name: Name for the gallery folder
+
+    Returns:
+        Tuple of (success, message)
+    """
+    try:
+        # Get gallery directory from environment variable or use default
+        gallery_dir = os.environ.get(
+            "DA3_GALLERY_DIR",
+            "workspace/gallery",
+        )
+        if not os.path.exists(gallery_dir):
+            os.makedirs(gallery_dir)
+
+        # Use provided name or create a unique name
+        if gallery_name is None or gallery_name.strip() == "":
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            gallery_name = f"reconstruction_{timestamp}"
+
+        gallery_path = os.path.join(gallery_dir, gallery_name)
+
+        # Check if directory already exists
+        if os.path.exists(gallery_path):
+            return False, f"Save failed: folder '{gallery_name}' already exists"
+
+        # Create the gallery directory
+        os.makedirs(gallery_path, exist_ok=True)
+
+        # Copy GLB file
+        glb_source = os.path.join(target_dir, "scene.glb")
+        glb_dest = os.path.join(gallery_path, "scene.glb")
+        if os.path.exists(glb_source):
+            shutil.copy2(glb_source, glb_dest)
+
+        # Copy depth visualization images
+        depth_vis_dir = os.path.join(target_dir, "depth_vis")
+        if os.path.exists(depth_vis_dir):
+            gallery_depth_vis = os.path.join(gallery_path, "depth_vis")
+            shutil.copytree(depth_vis_dir, gallery_depth_vis)
+
+        # Copy original images
+        images_source = os.path.join(target_dir, "images")
+        if os.path.exists(images_source):
+            gallery_images = os.path.join(gallery_path, "images")
+            shutil.copytree(images_source, gallery_images)
+
+        scene_preview_source = os.path.join(target_dir, "scene.jpg")
+        scene_preview_dest = os.path.join(gallery_path, "scene.jpg")
+        shutil.copy2(scene_preview_source, scene_preview_dest)
+
+        # Save metadata
+        metadata = {
+            "timestamp": datetime.now().strftime("%Y%m%d_%H%M%S"),
+            "num_images": len(processed_data) if processed_data else 0,
+            "gallery_name": gallery_name,
+        }
+
+        with open(os.path.join(gallery_path, "metadata.json"), "w") as f:
+            json.dump(metadata, f, indent=2)
+
+        print(f"Saved reconstruction to gallery: {gallery_path}")
+        return True, f"Save successful: saved to {gallery_path}"
+
+    except Exception as e:
+        print(f"Error saving to gallery: {e}")
+        return False, f"Save failed: {str(e)}"
+
+
+def get_scene_info(examples_dir: str) -> List[Dict[str, Any]]:
+    """
+    Get information about scenes in the examples directory.
+
+    Args:
+        examples_dir: Path to examples directory
+
+    Returns:
+        List of scene information dictionaries
+    """
+    import glob
+
+    scenes = []
+    if not os.path.exists(examples_dir):
+        return scenes
+
+    for scene_folder in sorted(os.listdir(examples_dir)):
+        scene_path = os.path.join(examples_dir, scene_folder)
+        if os.path.isdir(scene_path):
+            # Find all image files in the scene folder
+            image_extensions = ["*.jpg", "*.jpeg", "*.png", "*.bmp", "*.tiff", "*.tif"]
+            image_files = []
+            for ext in image_extensions:
+                image_files.extend(glob.glob(os.path.join(scene_path, ext)))
+                image_files.extend(glob.glob(os.path.join(scene_path, ext.upper())))
+
+            if image_files:
+                # Sort images and get the first one for thumbnail
+                image_files = sorted(image_files)
+                first_image = image_files[0]
+                num_images = len(image_files)
+
+                scenes.append(
+                    {
+                        "name": scene_folder,
+                        "path": scene_path,
+                        "thumbnail": first_image,
+                        "num_images": num_images,
+                        "image_files": image_files,
+                    }
+                )
+
+    return scenes
+
+
+# NOTE: cleanup was moved to a single canonical helper in
+# `depth_anything_3.utils.memory.cleanup_cuda_memory`.
+# Callers should import and call that directly instead of using this module.
+
+
+def get_logo_base64() -> Optional[str]:
+    """
+    Convert WAI logo to base64 for embedding in HTML.
+
+    Returns:
+        Base64 encoded logo string or None
+    """
+    import base64
+
+    logo_path = "examples/WAI-Logo/wai_logo.png"
+    try:
+        with open(logo_path, "rb") as img_file:
+            img_data = img_file.read()
+            base64_str = base64.b64encode(img_data).decode()
+            return f"data:image/png;base64,{base64_str}"
+    except FileNotFoundError:
+        return None

depth_anything_3/app/modules/visualization.py

@@ -0,0 +1,434 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Visualization module for Depth Anything 3 Gradio app.
+
+This module handles visualization updates, navigation, and measurement functionality.
+"""
+
+import os
+from typing import Any, Dict, List, Optional, Tuple
+import cv2
+import gradio as gr
+import numpy as np
+
+
+class VisualizationHandler:
+    """
+    Handles visualization updates and navigation for the Gradio app.
+    """
+
+    def __init__(self):
+        """Initialize the visualization handler."""
+
+    def update_view_selectors(
+        self, processed_data: Optional[Dict[int, Dict[str, Any]]]
+    ) -> Tuple[gr.Dropdown, gr.Dropdown]:
+        """
+        Update view selector dropdowns based on available views.
+
+        Args:
+            processed_data: Processed data dictionary
+
+        Returns:
+            Tuple of (depth_view_selector, measure_view_selector)
+        """
+        if processed_data is None or len(processed_data) == 0:
+            choices = ["View 1"]
+        else:
+            num_views = len(processed_data)
+            choices = [f"View {i + 1}" for i in range(num_views)]
+
+        return (
+            gr.Dropdown(choices=choices, value=choices[0]),  # depth_view_selector
+            gr.Dropdown(choices=choices, value=choices[0]),  # measure_view_selector
+        )
+
+    def get_view_data_by_index(
+        self, processed_data: Optional[Dict[int, Dict[str, Any]]], view_index: int
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Get view data by index, handling bounds.
+
+        Args:
+            processed_data: Processed data dictionary
+            view_index: Index of the view to get
+
+        Returns:
+            View data dictionary or None
+        """
+        if processed_data is None or len(processed_data) == 0:
+            return None
+
+        view_keys = list(processed_data.keys())
+        if view_index < 0 or view_index >= len(view_keys):
+            view_index = 0
+
+        return processed_data[view_keys[view_index]]
+
+    def update_depth_view(
+        self, processed_data: Optional[Dict[int, Dict[str, Any]]], view_index: int
+    ) -> Optional[str]:
+        """
+        Update depth view for a specific view index.
+
+        Args:
+            processed_data: Processed data dictionary
+            view_index: Index of the view to update
+
+        Returns:
+            Path to depth visualization image or None
+        """
+        view_data = self.get_view_data_by_index(processed_data, view_index)
+        if view_data is None or view_data.get("depth_image") is None:
+            return None
+
+        # Return the depth visualization image directly
+        return view_data["depth_image"]
+
+    def navigate_depth_view(
+        self,
+        processed_data: Optional[Dict[int, Dict[str, Any]]],
+        current_selector_value: str,
+        direction: int,
+    ) -> Tuple[str, Optional[str]]:
+        """
+        Navigate depth view (direction: -1 for previous, +1 for next).
+
+        Args:
+            processed_data: Processed data dictionary
+            current_selector_value: Current selector value
+            direction: Direction to navigate (-1 for previous, +1 for next)
+
+        Returns:
+            Tuple of (new_selector_value, depth_vis)
+        """
+        if processed_data is None or len(processed_data) == 0:
+            return "View 1", None
+
+        # Parse current view number
+        try:
+            current_view = int(current_selector_value.split()[1]) - 1
+        except:  # noqa
+            current_view = 0
+
+        num_views = len(processed_data)
+        new_view = (current_view + direction) % num_views
+
+        new_selector_value = f"View {new_view + 1}"
+        depth_vis = self.update_depth_view(processed_data, new_view)
+
+        return new_selector_value, depth_vis
+
+    def update_measure_view(
+        self, processed_data: Optional[Dict[int, Dict[str, Any]]], view_index: int
+    ) -> Tuple[Optional[np.ndarray], Optional[np.ndarray], List]:
+        """
+        Update measure view for a specific view index.
+
+        Args:
+            processed_data: Processed data dictionary
+            view_index: Index of the view to update
+
+        Returns:
+            Tuple of (measure_image, depth_right_half, measure_points)
+        """
+        view_data = self.get_view_data_by_index(processed_data, view_index)
+        if view_data is None:
+            return None, None, []  # image, depth_right_half, measure_points
+
+        # Get the processed (resized) image
+        if "image" in view_data and view_data["image"] is not None:
+            image = view_data["image"].copy()
+        else:
+            return None, None, []
+
+        # Ensure image is in uint8 format
+        if image.dtype != np.uint8:
+            if image.max() <= 1.0:
+                image = (image * 255).astype(np.uint8)
+            else:
+                image = image.astype(np.uint8)
+
+        # Extract right half of the depth visualization (pure depth part)
+        depth_image_path = view_data.get("depth_image", None)
+        depth_right_half = None
+
+        if depth_image_path and os.path.exists(depth_image_path):
+            try:
+                # Load the combined depth visualization image
+                depth_combined = cv2.imread(depth_image_path)
+                depth_combined = cv2.cvtColor(depth_combined, cv2.COLOR_BGR2RGB)
+                if depth_combined is not None:
+                    height, width = depth_combined.shape[:2]
+                    # Extract right half (depth visualization part)
+                    depth_right_half = depth_combined[:, width // 2 :]
+            except Exception as e:
+                print(f"Error extracting depth right half: {e}")
+
+        return image, depth_right_half, []
+
+    def navigate_measure_view(
+        self,
+        processed_data: Optional[Dict[int, Dict[str, Any]]],
+        current_selector_value: str,
+        direction: int,
+    ) -> Tuple[str, Optional[np.ndarray], Optional[str], List]:
+        """
+        Navigate measure view (direction: -1 for previous, +1 for next).
+
+        Args:
+            processed_data: Processed data dictionary
+            current_selector_value: Current selector value
+            direction: Direction to navigate (-1 for previous, +1 for next)
+
+        Returns:
+            Tuple of (new_selector_value, measure_image, depth_image_path, measure_points)
+        """
+        if processed_data is None or len(processed_data) == 0:
+            return "View 1", None, None, []
+
+        # Parse current view number
+        try:
+            current_view = int(current_selector_value.split()[1]) - 1
+        except:  # noqa
+            current_view = 0
+
+        num_views = len(processed_data)
+        new_view = (current_view + direction) % num_views
+
+        new_selector_value = f"View {new_view + 1}"
+        measure_image, depth_right_half, measure_points = self.update_measure_view(
+            processed_data, new_view
+        )
+
+        return new_selector_value, measure_image, depth_right_half, measure_points
+
+    def populate_visualization_tabs(
+        self, processed_data: Optional[Dict[int, Dict[str, Any]]]
+    ) -> Tuple[Optional[str], Optional[np.ndarray], Optional[str], List]:
+        """
+        Populate the depth and measure tabs with processed data.
+
+        Args:
+            processed_data: Processed data dictionary
+
+        Returns:
+            Tuple of (depth_vis, measure_img, depth_image_path, measure_points)
+        """
+        if processed_data is None or len(processed_data) == 0:
+            return None, None, None, []
+
+        # Use update function to get depth visualization
+        depth_vis = self.update_depth_view(processed_data, 0)
+        measure_img, depth_right_half, _ = self.update_measure_view(processed_data, 0)
+
+        return depth_vis, measure_img, depth_right_half, []
+
+    def reset_measure(
+        self, processed_data: Optional[Dict[int, Dict[str, Any]]]
+    ) -> Tuple[Optional[np.ndarray], List, str]:
+        """
+        Reset measure points.
+
+        Args:
+            processed_data: Processed data dictionary
+
+        Returns:
+            Tuple of (image, measure_points, text)
+        """
+        if processed_data is None or len(processed_data) == 0:
+            return None, [], ""
+
+        # Return the first view image
+        first_view = list(processed_data.values())[0]
+        return first_view["image"], [], ""
+
+    def measure(
+        self,
+        processed_data: Optional[Dict[int, Dict[str, Any]]],
+        measure_points: List,
+        current_view_selector: str,
+        event: gr.SelectData,
+    ) -> List:
+        """
+        Handle measurement on images.
+
+        Args:
+            processed_data: Processed data dictionary
+            measure_points: List of current measure points
+            current_view_selector: Current view selector value
+            event: Gradio select event
+
+        Returns:
+            List of [image, depth_right_half, measure_points, text]
+        """
+        try:
+            print(f"Measure function called with selector: {current_view_selector}")
+
+            if processed_data is None or len(processed_data) == 0:
+                return [None, [], "No data available"]
+
+            # Use the currently selected view instead of always using the first view
+            try:
+                current_view_index = int(current_view_selector.split()[1]) - 1
+            except:  # noqa
+                current_view_index = 0
+
+            print(f"Using view index: {current_view_index}")
+
+            # Get view data safely
+            if current_view_index < 0 or current_view_index >= len(processed_data):
+                current_view_index = 0
+
+            view_keys = list(processed_data.keys())
+            current_view = processed_data[view_keys[current_view_index]]
+
+            if current_view is None:
+                return [None, [], "No view data available"]
+
+            point2d = event.index[0], event.index[1]
+            print(f"Clicked point: {point2d}")
+
+            measure_points.append(point2d)
+
+            # Get image and depth visualization
+            image, depth_right_half, _ = self.update_measure_view(
+                processed_data, current_view_index
+            )
+            if image is None:
+                return [None, [], "No image available"]
+
+            image = image.copy()
+
+            # Ensure image is in uint8 format for proper cv2 operations
+            try:
+                if image.dtype != np.uint8:
+                    if image.max() <= 1.0:
+                        # Image is in [0, 1] range, convert to [0, 255]
+                        image = (image * 255).astype(np.uint8)
+                    else:
+                        # Image is already in [0, 255] range
+                        image = image.astype(np.uint8)
+            except Exception as e:
+                print(f"Image conversion error: {e}")
+                return [None, [], f"Image conversion error: {e}"]
+
+            # Draw circles for points
+            try:
+                for p in measure_points:
+                    if 0 <= p[0] < image.shape[1] and 0 <= p[1] < image.shape[0]:
+                        image = cv2.circle(image, p, radius=5, color=(255, 0, 0), thickness=2)
+            except Exception as e:
+                print(f"Drawing error: {e}")
+                return [None, [], f"Drawing error: {e}"]
+
+            # Get depth information from processed_data
+            depth_text = ""
+            try:
+                for i, p in enumerate(measure_points):
+                    if (
+                        current_view["depth"] is not None
+                        and 0 <= p[1] < current_view["depth"].shape[0]
+                        and 0 <= p[0] < current_view["depth"].shape[1]
+                    ):
+                        d = current_view["depth"][p[1], p[0]]
+                        depth_text += f"- **P{i + 1} depth: {d:.2f}m**\n"
+                    else:
+                        depth_text += f"- **P{i + 1}: Click position ({p[0]}, {p[1]}) - No depth information**\n"  # noqa: E501
+            except Exception as e:
+                print(f"Depth text error: {e}")
+                depth_text = f"Error computing depth: {e}\n"
+
+            if len(measure_points) == 2:
+                try:
+                    point1, point2 = measure_points
+                    # Draw line
+                    if (
+                        0 <= point1[0] < image.shape[1]
+                        and 0 <= point1[1] < image.shape[0]
+                        and 0 <= point2[0] < image.shape[1]
+                        and 0 <= point2[1] < image.shape[0]
+                    ):
+                        image = cv2.line(image, point1, point2, color=(255, 0, 0), thickness=2)
+
+                    # Compute 3D distance using depth information and camera intrinsics
+                    distance_text = "- **Distance: Unable to calculate 3D distance**"
+                    if (
+                        current_view["depth"] is not None
+                        and 0 <= point1[1] < current_view["depth"].shape[0]
+                        and 0 <= point1[0] < current_view["depth"].shape[1]
+                        and 0 <= point2[1] < current_view["depth"].shape[0]
+                        and 0 <= point2[0] < current_view["depth"].shape[1]
+                    ):
+                        try:
+                            # Get depth values at the two points
+                            d1 = current_view["depth"][point1[1], point1[0]]
+                            d2 = current_view["depth"][point2[1], point2[0]]
+
+                            # Convert 2D pixel coordinates to 3D world coordinates
+                            if current_view["intrinsics"] is not None:
+                                # Get camera intrinsics
+                                K = current_view["intrinsics"]  # 3x3 intrinsic matrix
+                                fx, fy = K[0, 0], K[1, 1]  # focal lengths
+                                cx, cy = K[0, 2], K[1, 2]  # principal point
+
+                                # Convert pixel coordinates to normalized camera coordinates
+                                # Point 1: (u1, v1) -> (x1, y1, z1)
+                                u1, v1 = point1[0], point1[1]
+                                x1 = (u1 - cx) * d1 / fx
+                                y1 = (v1 - cy) * d1 / fy
+                                z1 = d1
+
+                                # Point 2: (u2, v2) -> (x2, y2, z2)
+                                u2, v2 = point2[0], point2[1]
+                                x2 = (u2 - cx) * d2 / fx
+                                y2 = (v2 - cy) * d2 / fy
+                                z2 = d2
+
+                                # Calculate 3D Euclidean distance
+                                p1_3d = np.array([x1, y1, z1])
+                                p2_3d = np.array([x2, y2, z2])
+                                distance_3d = np.linalg.norm(p1_3d - p2_3d)
+
+                                distance_text = f"- **Distance: {distance_3d:.2f}m**"
+                            else:
+                                # Fallback to simplified calculation if no intrinsics
+                                pixel_distance = np.sqrt(
+                                    (point1[0] - point2[0]) ** 2 + (point1[1] - point2[1]) ** 2
+                                )
+                                avg_depth = (d1 + d2) / 2
+                                scale_factor = avg_depth / 1000  # Rough scaling factor
+                                estimated_3d_distance = pixel_distance * scale_factor
+                                distance_text = f"- **Distance: {estimated_3d_distance:.2f}m (estimated, no intrinsics)**"  # noqa: E501
+
+                        except Exception as e:
+                            print(f"Distance computation error: {e}")
+                            distance_text = f"- **Distance computation error: {e}**"
+
+                    measure_points = []
+                    text = depth_text + distance_text
+                    print(f"Measurement complete: {text}")
+                    return [image, depth_right_half, measure_points, text]
+                except Exception as e:
+                    print(f"Final measurement error: {e}")
+                    return [None, [], f"Measurement error: {e}"]
+            else:
+                print(f"Single point measurement: {depth_text}")
+                return [image, depth_right_half, measure_points, depth_text]
+
+        except Exception as e:
+            print(f"Overall measure function error: {e}")
+            return [None, [], f"Measure function error: {e}"]

depth_anything_3/bench/__init__.py

@@ -0,0 +1,45 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Depth Anything 3 Benchmark Evaluation Module.
+
+This module provides tools for evaluating DepthAnything3 model on various benchmark datasets.
+Currently supported datasets:
+- DTU (3D Reconstruction)
+- DTU-64 (Pose Evaluation Only)
+- ETH3D (3D Reconstruction)
+- 7Scenes (3D Reconstruction)
+- ScanNet++ (3D Reconstruction)
+- HiRoom (3D Reconstruction)
+
+Supported evaluation modes:
+- pose: Camera pose estimation evaluation
+- recon_unposed: 3D reconstruction with predicted poses
+- recon_posed: 3D reconstruction with ground truth poses
+"""
+
+from depth_anything_3.bench.registries import MV_REGISTRY, MONO_REGISTRY
+
+
+def __getattr__(name):
+    """Lazy import to avoid circular import when running as __main__."""
+    if name == "Evaluator":
+        from depth_anything_3.bench.evaluator import Evaluator
+        return Evaluator
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = ["Evaluator", "MV_REGISTRY", "MONO_REGISTRY"]
+

depth_anything_3/bench/configs/eval_bench.yaml

@@ -0,0 +1,98 @@
+# DepthAnything3 Benchmark Evaluation Configuration
+# 
+# This config can be loaded and overridden via command line.
+# Example: python -m depth_anything_3.bench.evaluator --model /path/to/model --work_dir /path/to/workspace
+#
+# See depth_anything_3.cfg for config utility functions.
+
+# ==============================================================================
+# Model Configuration
+# ==============================================================================
+model:
+  # Path to model checkpoint or HuggingFace model ID
+  path: depth-anything/DA3-GIANT
+
+# ==============================================================================
+# Workspace Configuration
+# ==============================================================================
+workspace:
+  # Working directory for outputs (model results, metrics, etc.)
+  work_dir: ./workspace/evaluation
+
+# ==============================================================================
+# Evaluation Configuration
+# ==============================================================================
+eval:
+  # Datasets to evaluate
+  # Options: dtu, dtu64, eth3d, 7scenes (sevenscenes), scannetpp, hiroom
+  datasets:
+    - eth3d
+    - 7scenes
+    - scannetpp
+    - hiroom
+    - dtu
+    - dtu64
+
+  # Evaluation modes
+  # Options: pose, recon_unposed, recon_posed, view_syn
+  modes:
+    - pose
+    - recon_unposed
+    - recon_posed
+
+  # Reference view selection strategy for inference
+  # Options: first, saddle_balanced, auto, mid
+  ref_view_strategy: "first"
+
+  # Specific scenes to evaluate (null = all scenes)
+  # Example: [courtyard, relief] for eth3d
+  scenes: null
+
+  # Maximum number of frames per scene (for sampling)
+  # If a scene has more frames, randomly sample to this limit.
+  # Set to -1 to disable sampling.
+  max_frames: 100
+
+  # Only run evaluation (skip inference)
+  eval_only: false
+
+  # Only print saved metrics (skip inference and evaluation)
+  print_only: false
+
+# ==============================================================================
+# Inference Configuration
+# ==============================================================================
+inference:
+  # Number of parallel workers for TSDF fusion
+  num_fusion_workers: 4
+
+  # Enable debug mode with verbose output
+  debug: false
+
+# ==============================================================================
+# Preset Configurations
+# ==============================================================================
+# These can be activated via command line: --preset full_eval
+
+presets:
+  # Full evaluation on all 6 datasets
+  full_eval:
+    datasets: [eth3d, 7scenes, scannetpp, hiroom, dtu, dtu64]
+    modes: [pose, recon_unposed, recon_posed]
+
+  # Pose-only evaluation
+  pose_only:
+    datasets: [eth3d, 7scenes, scannetpp, hiroom, dtu64]
+    modes: [pose]
+
+  # Reconstruction-only evaluation (5 datasets, excluding dtu64)
+  recon_only:
+    datasets: [eth3d, 7scenes, scannetpp, hiroom, dtu]
+    modes: [recon_unposed, recon_posed]
+
+  # Quick test (single scene per dataset)
+  quick_test:
+    datasets: [eth3d]
+    modes: [pose, recon_unposed]
+    scenes: [courtyard]
+

depth_anything_3/bench/dataset.py

@@ -0,0 +1,136 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Base dataset class for benchmark evaluation.
+
+All dataset implementations should inherit from this class and implement
+the required abstract methods.
+"""
+
+import os
+import time
+from abc import abstractmethod
+from typing import Dict as TDict
+
+import numpy as np
+import torch
+from addict import Dict
+
+from depth_anything_3.bench.utils import compute_pose
+from depth_anything_3.utils.geometry import as_homogeneous
+
+
+def _wait_for_file_ready(path: str, timeout: float = 3.0, interval: float = 0.2) -> None:
+    """Wait until file size stabilizes for 2 consecutive checks."""
+    last_size = -1
+    stable_count = 0
+    start = time.time()
+    while time.time() - start < timeout:
+        time.sleep(interval)
+        size = os.path.getsize(path)
+        if size == last_size and size > 0:
+            stable_count += 1
+            if stable_count >= 2:  # Need 2 consecutive stable checks
+                return
+        else:
+            stable_count = 0
+        last_size = size
+
+
+class Dataset:
+    """
+    Base class for all benchmark datasets.
+
+    Subclasses must implement:
+        - SCENES: List of scene identifiers
+        - data_root: Path to dataset root
+        - get_data(scene): Return scene data (images, intrinsics, extrinsics, etc.)
+        - eval3d(scene, fuse_path): Evaluate 3D reconstruction
+        - fuse3d(scene, result_path, fuse_path, mode): Fuse depth maps into point cloud
+
+    Optional overrides:
+        - eval_pose(scene, result_path): Evaluate pose estimation (default provided)
+    """
+
+    # Subclasses should define these
+    SCENES: list = []
+    data_root: str = ""
+
+    def __init__(self):
+        pass
+
+    def eval_pose(self, scene: str, result_path: str) -> TDict[str, float]:
+        """
+        Evaluate camera pose estimation accuracy.
+
+        Args:
+            scene: Scene identifier
+            result_path: Path to .npz file containing predicted extrinsics
+
+        Returns:
+            Dict with pose metrics (auc30, auc15, auc05, auc03)
+        """
+        _wait_for_file_ready(result_path)
+        pred = np.load(result_path)
+        gt = self.get_data(scene)
+        return compute_pose(
+            torch.from_numpy(as_homogeneous(pred["extrinsics"])),
+            torch.from_numpy(as_homogeneous(gt["extrinsics"])),
+        )
+
+    @abstractmethod
+    def get_data(self, scene: str) -> Dict:
+        """
+        Get scene data including images, camera parameters, and auxiliary info.
+
+        Args:
+            scene: Scene identifier
+
+        Returns:
+            Dict with:
+                - image_files: List[str] - paths to images
+                - extrinsics: np.ndarray [N, 4, 4] - camera extrinsics (world-to-camera)
+                - intrinsics: np.ndarray [N, 3, 3] - camera intrinsics
+                - aux: Dict - auxiliary data (masks, GT paths, etc.)
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def eval3d(self, scene: str, fuse_path: str) -> TDict[str, float]:
+        """
+        Evaluate 3D reconstruction quality against ground truth.
+
+        Args:
+            scene: Scene identifier
+            fuse_path: Path to fused point cloud (.ply)
+
+        Returns:
+            Dict with reconstruction metrics (e.g., acc, comp, overall)
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def fuse3d(self, scene: str, result_path: str, fuse_path: str, mode: str) -> None:
+        """
+        Fuse per-view depth maps into a single point cloud.
+
+        Args:
+            scene: Scene identifier
+            result_path: Path to .npz file with predicted depths and poses
+            fuse_path: Output path for fused point cloud (.ply)
+            mode: Fusion mode ("recon_unposed" or "recon_posed")
+        """
+        raise NotImplementedError
+

depth_anything_3/bench/datasets/__init__.py

@@ -0,0 +1,21 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Benchmark dataset implementations.
+
+Datasets are auto-registered via decorators when imported.
+Add new dataset files here and they will be automatically discovered.
+"""
+

depth_anything_3/bench/datasets/dtu.py

@@ -0,0 +1,681 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+DTU Benchmark dataset implementation.
+
+DTU is a multi-view stereo benchmark for 3D reconstruction evaluation.
+Reference: https://roboimagedata.compute.dtu.dk/
+
+Note: DepthAnything3 was never trained on any images from DTU.
+"""
+
+import glob
+import os
+from typing import Dict as TDict, List
+
+import numpy as np
+import open3d as o3d
+import torch
+import torch.nn.functional as F
+from addict import Dict
+from PIL import Image
+from plyfile import PlyData
+from scipy.io import loadmat
+from sklearn import neighbors as skln
+from tqdm import tqdm
+
+from depth_anything_3.bench.dataset import Dataset
+from depth_anything_3.bench.registries import MONO_REGISTRY, MV_REGISTRY
+from depth_anything_3.utils.constants import (
+    DTU_DIST_THRESH,
+    DTU_EVAL_DATA_ROOT,
+    DTU_MAX_POINTS,
+    DTU_NUM_CONSIST,
+    DTU_SCENES,
+)
+from depth_anything_3.utils.pose_align import align_poses_umeyama
+
+
+@MV_REGISTRY.register(name="dtu")
+@MONO_REGISTRY.register(name="dtu")
+class DTU(Dataset):
+    """
+    DTU Benchmark dataset wrapper for DepthAnything3 evaluation.
+
+    Supports:
+        - Camera pose estimation evaluation (AUC metrics)
+        - 3D reconstruction evaluation (accuracy, completeness, overall)
+        - Point cloud fusion from depth maps
+
+    The dataset uses MVSNet evaluation protocol:
+    https://drive.google.com/file/d/1rX0EXlUL4prRxrRu2DgLJv2j7-tpUD4D/view
+    """
+
+    data_root = DTU_EVAL_DATA_ROOT
+    SCENES = DTU_SCENES
+
+    # Evaluation/triangulation hyperparameters from constants
+    dist_thresh = DTU_DIST_THRESH
+    num_consist = DTU_NUM_CONSIST
+
+    # ------------------------------
+    # Public API
+    # ------------------------------
+
+    def read_cam_file(self, filename: str) -> tuple:
+        """
+        Read DTU camera file containing extrinsics and intrinsics.
+
+        Args:
+            filename: Path to camera text file
+
+        Returns:
+            Tuple of (intrinsics [3,3], extrinsics [4,4])
+        """
+        with open(filename) as f:
+            lines = [line.rstrip() for line in f.readlines()]
+        extrinsics = np.fromstring(" ".join(lines[1:5]), dtype=np.float32, sep=" ").reshape((4, 4))
+        intrinsics = np.fromstring(" ".join(lines[7:10]), dtype=np.float32, sep=" ").reshape((3, 3))
+        return intrinsics, extrinsics
+
+    def get_data(self, scene: str) -> Dict:
+        """
+        Collect per-view image paths, intrinsics/extrinsics, and GT masks.
+
+        Args:
+            scene: Scene identifier (e.g., "scan1")
+
+        Returns:
+            Dict with:
+                - image_files: List[str] - paths to images
+                - extrinsics: np.ndarray [N, 4, 4]
+                - intrinsics: np.ndarray [N, 3, 3]
+                - aux.mask_files: List[str] - paths to depth masks
+        """
+        rgb_folder = os.path.join(self.data_root, "Rectified", scene)
+        camera_folder = os.path.join(self.data_root, "Cameras")
+
+        files = sorted(glob.glob(os.path.join(rgb_folder, "*.png")))
+        # Reorder: place index 33 first (reference view convention)
+        files = [files[33]] + files[:33] + files[34:]
+
+        out = Dict(
+            {
+                "image_files": files,
+                "extrinsics": [],
+                "intrinsics": [],
+                "aux": Dict({"mask_files": []}),
+            }
+        )
+
+        for rgb_file in files:
+            basename = os.path.basename(rgb_file)
+            file_idx = basename.split("_")[1]
+            cam_idx = depth_idx = int(file_idx) - 1
+
+            mask_file = self._depth_mask_path(scene, depth_idx)
+            proj_mat_filename = os.path.join(camera_folder, f"{cam_idx:0>8}_cam.txt")
+
+            ixt, ext = self.read_cam_file(proj_mat_filename)
+            out.extrinsics.append(ext)
+            out.intrinsics.append(ixt)
+            out.aux.mask_files.append(mask_file)
+
+        out.extrinsics = np.asarray(out.extrinsics, dtype=np.float32)
+        out.intrinsics = np.asarray(out.intrinsics, dtype=np.float32)
+        return out
+
+    def get_3dgtpath(self, scene: str) -> str:
+        """Get path to ground truth point cloud for a scene."""
+        scene_id = int(scene[4:])
+        return os.path.join(self.data_root, f"Points/stl/stl{scene_id:03}_total.ply")
+
+    def eval3d(self, scene: str, fuse_path: str, use_gpu: bool = False) -> TDict[str, float]:
+        """
+        Evaluate fused point cloud against DTU GT with ObsMask/Plane.
+
+        Args:
+            scene: Scene identifier
+            fuse_path: Path to fused point cloud
+            use_gpu: If True, use GPU-accelerated distance computation (faster but may have minor numerical differences)
+
+        Returns:
+            Dict with metrics: {"comp": float, "acc": float, "overall": float}
+        """
+        scene_id = int(scene[4:])
+        gt_ply = os.path.join(self.data_root, f"Points/stl/stl{scene_id:03}_total.ply")
+        mask_file = os.path.join(
+            self.data_root, f"SampleSet/mvs_data/ObsMask/ObsMask{scene_id}_10.mat"
+        )
+        plane_file = os.path.join(
+            self.data_root, f"SampleSet/mvs_data/ObsMask/Plane{scene_id}.mat"
+        )
+        result = self._evaluate_reconstruction(
+            scene, fuse_path, gt_ply, mask_file, plane_file, use_gpu=use_gpu
+        )
+        return {"comp": result[0], "acc": result[1], "overall": result[2]}
+
+    def load_masks(self, mask_files: List[str]) -> np.ndarray:
+        """
+        Load DTU depth validity masks.
+
+        Args:
+            mask_files: List of paths to mask images
+
+        Returns:
+            Boolean array [N, H, W] indicating valid depth regions
+        """
+        masks = []
+        for mask_file in mask_files:
+            mask = Image.open(mask_file)
+            mask = np.array(mask, dtype=np.float32)
+            masks.append(mask > 10)
+        return np.asarray(masks)
+
+    def fuse3d(self, scene: str, result_path: str, fuse_path: str, mode: str) -> None:
+        """
+        Fuse per-view depths into a point cloud and save to PLY.
+
+        Args:
+            scene: Scene identifier (e.g., "scan114")
+            result_path: Path to npz file containing predicted depths/poses
+            fuse_path: Output path for fused point cloud (.ply)
+            mode: "recon_unposed" or "recon_posed"
+        """
+        gt_data = self.get_data(scene)
+        pred_data = Dict({k: v for k, v in np.load(result_path).items()})
+        masks = self.load_masks(gt_data.aux.mask_files)
+
+        if mode == "recon_unposed":
+            depths, intrinsics, extrinsics = self._prep_unposed(pred_data, gt_data, masks)
+        elif mode == "recon_posed":
+            depths, intrinsics, extrinsics = self._prep_posed(pred_data, gt_data, masks)
+        else:
+            raise ValueError(f"Invalid mode: {mode}")
+
+        proj_mat = self._build_proj_mats(intrinsics, extrinsics)
+
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        dtype = torch.float32
+        depths_t = torch.from_numpy(depths).to(device=device, dtype=dtype).unsqueeze(1)
+        proj_t = torch.from_numpy(proj_mat).to(device=device, dtype=dtype)
+        height, width = depths_t.shape[-2:]
+
+        points: List[np.ndarray] = []
+        for idx in range(len(gt_data.image_files)):
+            if mode == "recon_unposed":
+                # Simple unfiltered back-projection per frame
+                cur_p_pcd = self._generate_points_from_depth(
+                    depths_t[idx : idx + 1], proj_t[idx : idx + 1]
+                )
+                mask = (depths_t[idx : idx + 1] > 0.001).squeeze()
+                cur_p_pcd = cur_p_pcd[:, :, mask]
+                no_filter_pc = cur_p_pcd.squeeze(0).permute(1, 0).cpu().numpy()
+                points.append(no_filter_pc)
+            else:  # recon_posed
+                final_pc = self._fuse_consistent_points(depths_t, proj_t, idx, height, width)
+                points.append(final_pc)
+
+        # Concatenate and optionally downsample to hard cap
+        points_np = np.concatenate(points, axis=0)
+        points_np = self._cap_points(points_np, max_points=DTU_MAX_POINTS)
+
+        os.makedirs(os.path.dirname(fuse_path), exist_ok=True)
+        pcd = o3d.geometry.PointCloud()
+        pcd.points = o3d.utility.Vector3dVector(points_np)
+        o3d.io.write_point_cloud(fuse_path, pcd)
+
+    # ------------------------------
+    # Geometry helpers
+    # ------------------------------
+
+    def _generate_points_from_depth(
+        self, depth: torch.Tensor, proj: torch.Tensor
+    ) -> torch.Tensor:
+        """
+        Back-project depth map into 3D world coordinates.
+
+        Args:
+            depth: Depth tensor [B, 1, H, W]
+            proj: Projection matrix [B, 4, 4] = [[K@R, K@t], [0,0,0,1]]
+
+        Returns:
+            Point cloud tensor [B, 3, H, W]
+        """
+        batch, height, width = depth.shape[0], depth.shape[2], depth.shape[3]
+        inv_proj = torch.inverse(proj)
+        rot = inv_proj[:, :3, :3]
+        trans = inv_proj[:, :3, 3:4]
+
+        y, x = torch.meshgrid(
+            [
+                torch.arange(0, height, dtype=torch.float32, device=depth.device),
+                torch.arange(0, width, dtype=torch.float32, device=depth.device),
+            ],
+            indexing="ij",
+        )
+        y, x = y.contiguous(), x.contiguous()
+        y, x = y.view(height * width), x.view(height * width)
+        xyz = torch.stack((x, y, torch.ones_like(x)))
+        xyz = torch.unsqueeze(xyz, 0).repeat(batch, 1, 1)
+        rot_xyz = torch.matmul(rot, xyz)
+        rot_depth_xyz = rot_xyz * depth.view(batch, 1, -1)
+        proj_xyz = rot_depth_xyz + trans.view(batch, 3, 1)
+        return proj_xyz.view(batch, 3, height, width)
+
+    def _homo_warping(
+        self,
+        src_fea: torch.Tensor,
+        src_proj: torch.Tensor,
+        ref_proj: torch.Tensor,
+        depth_values: torch.Tensor,
+    ) -> torch.Tensor:
+        """
+        Homography warping for multi-view consistency checking.
+
+        Args:
+            src_fea: Source features [B, C, H, W]
+            src_proj: Source projection [B, 4, 4]
+            ref_proj: Reference projection [B, 4, 4]
+            depth_values: Depth values [B, Ndepth] or [B, Ndepth, H, W]
+
+        Returns:
+            Warped features [B, C, H, W]
+        """
+        batch, channels = src_fea.shape[0], src_fea.shape[1]
+        height, width = src_fea.shape[2], src_fea.shape[3]
+
+        with torch.no_grad():
+            proj = torch.matmul(src_proj, torch.inverse(ref_proj))
+            rot = proj[:, :3, :3]
+            trans = proj[:, :3, 3:4]
+
+            y, x = torch.meshgrid(
+                [
+                    torch.arange(0, height, dtype=torch.float32, device=src_fea.device),
+                    torch.arange(0, width, dtype=torch.float32, device=src_fea.device),
+                ],
+                indexing="ij",
+            )
+            y, x = y.contiguous(), x.contiguous()
+            y, x = y.view(height * width), x.view(height * width)
+            xyz = torch.stack((x, y, torch.ones_like(x)))
+            xyz = torch.unsqueeze(xyz, 0).repeat(batch, 1, 1)
+            rot_xyz = torch.matmul(rot, xyz)
+
+            rot_depth_xyz = rot_xyz.unsqueeze(2) * depth_values.view(-1, 1, 1, height * width)
+            proj_xyz = rot_depth_xyz + trans.view(batch, 3, 1, 1)
+            proj_xy = proj_xyz[:, :2, :, :] / proj_xyz[:, 2:3, :, :]
+            proj_x_normalized = proj_xy[:, 0, :, :] / ((width - 1) / 2) - 1
+            proj_y_normalized = proj_xy[:, 1, :, :] / ((height - 1) / 2) - 1
+            grid = torch.stack((proj_x_normalized, proj_y_normalized), dim=3)
+
+        warped_src_fea = F.grid_sample(
+            src_fea,
+            grid.view(batch, height, width, 2),
+            mode="bilinear",
+            padding_mode="zeros",
+            align_corners=True,
+        )
+        return warped_src_fea.view(batch, channels, height, width)
+
+    def _filter_depth(
+        self,
+        ref_depth: torch.Tensor,
+        src_depths: torch.Tensor,
+        ref_proj: torch.Tensor,
+        src_projs: torch.Tensor,
+    ) -> tuple:
+        """
+        Compute geometric consistency between reference and source depths.
+
+        Args:
+            ref_depth: Reference depth [1, 1, H, W]
+            src_depths: Source depths [B, 1, H, W]
+            ref_proj: Reference projection [1, 4, 4]
+            src_projs: Source projections [B, 4, 4]
+
+        Returns:
+            Tuple of (ref_pc, aligned_pcs, dist)
+        """
+        ref_pc = self._generate_points_from_depth(ref_depth, ref_proj)
+        src_pcs = self._generate_points_from_depth(src_depths, src_projs)
+        aligned_pcs = self._homo_warping(src_pcs, src_projs, ref_proj, ref_depth)
+        x_2 = (ref_pc[:, 0] - aligned_pcs[:, 0]) ** 2
+        y_2 = (ref_pc[:, 1] - aligned_pcs[:, 1]) ** 2
+        z_2 = (ref_pc[:, 2] - aligned_pcs[:, 2]) ** 2
+        dist = torch.sqrt(x_2 + y_2 + z_2).unsqueeze(1)
+        return ref_pc, aligned_pcs, dist
+
+    def _extract_points(
+        self, pc: torch.Tensor, mask: torch.Tensor, rgb: np.ndarray = None
+    ) -> np.ndarray:
+        """Extract masked points from a dense grid."""
+        pc = pc.cpu().numpy()
+        mask = mask.cpu().numpy().reshape(-1)
+        pc = pc.reshape(-1, 3)
+        points = pc[np.where(mask)]
+        if rgb is not None:
+            rgb = rgb.reshape(-1, 3)
+            colors = rgb[np.where(mask)]
+            return np.concatenate([points, colors], axis=1)
+        return points
+
+    # ------------------------------
+    # 3D Reconstruction Evaluation
+    # ------------------------------
+
+    def _evaluate_reconstruction(
+        self,
+        scanid: str,
+        pred_ply: str,
+        gt_ply: str,
+        mask_file: str,
+        plane_file: str,
+        down_dense: float = 0.2,
+        patch: int = 60,
+        max_dist: int = 20,
+        use_gpu: bool = False,
+    ) -> tuple:
+        """
+        Compute accuracy, completeness, and overall metrics for one scan.
+
+        Args:
+            scanid: Scan identifier
+            pred_ply: Predicted point cloud path or array
+            gt_ply: Ground truth point cloud path or array
+            mask_file: ObsMask file path
+            plane_file: Plane file path
+            down_dense: Downsample density (min distance between points)
+            patch: Patch size for boundary
+            max_dist: Outlier threshold in mm
+            use_gpu: If True, use GPU-accelerated distance computation
+
+        Returns:
+            Tuple of (mean_d2s, mean_s2d, overall)
+        """
+        thresh = down_dense
+
+        # Load and downsample predicted point cloud
+        data_pcd = self._read_ply(pred_ply) if isinstance(pred_ply, str) else pred_ply
+        # Use fixed seed for reproducibility
+        shuffle_rng = np.random.default_rng(seed=42)
+        shuffle_rng.shuffle(data_pcd, axis=0)
+
+        # Downsample point cloud
+        nn_engine = skln.NearestNeighbors(
+            n_neighbors=1, radius=thresh, algorithm="kd_tree", n_jobs=-1
+        )
+        nn_engine.fit(data_pcd)
+        rnn_idxs = nn_engine.radius_neighbors(data_pcd, radius=thresh, return_distance=False)
+        mask = np.ones(data_pcd.shape[0], dtype=np.bool_)
+        for curr, idxs in enumerate(rnn_idxs):
+            if mask[curr]:
+                mask[idxs] = 0
+                mask[curr] = 1
+        data_down = data_pcd[mask]
+
+        # Restrict to observed volume (ObsMask)
+        obs_mask_file = loadmat(mask_file)
+        ObsMask, BB, Res = (obs_mask_file[attr] for attr in ["ObsMask", "BB", "Res"])
+        BB = BB.astype(np.float32)
+
+        inbound = ((data_down >= BB[:1] - patch) & (data_down < BB[1:] + patch * 2)).sum(
+            axis=-1
+        ) == 3
+        data_in = data_down[inbound]
+
+        data_grid = np.around((data_in - BB[:1]) / Res).astype(np.int32)
+        grid_inbound = ((data_grid >= 0) & (data_grid < np.expand_dims(ObsMask.shape, 0))).sum(
+            axis=-1
+        ) == 3
+        data_grid_in = data_grid[grid_inbound]
+        in_obs = ObsMask[data_grid_in[:, 0], data_grid_in[:, 1], data_grid_in[:, 2]].astype(
+            np.bool_
+        )
+        data_in_obs = data_in[grid_inbound][in_obs]
+
+        # Compute accuracy (pred -> GT) and completeness (GT -> pred)
+        stl = self._read_ply(gt_ply) if isinstance(gt_ply, str) else gt_ply
+
+        if use_gpu and torch.cuda.is_available():
+            # GPU-accelerated distance computation
+            mean_d2s = self._knn_dist_gpu(data_in_obs, stl, max_dist)
+        else:
+            # CPU version (original, for exact reproduction)
+            nn_engine.fit(stl)
+            dist_d2s, _ = nn_engine.kneighbors(data_in_obs, n_neighbors=1, return_distance=True)
+            mean_d2s = dist_d2s[dist_d2s < max_dist].mean()
+
+        ground_plane = loadmat(plane_file)["P"]
+        stl_hom = np.concatenate([stl, np.ones_like(stl[:, :1])], -1)
+        above = (ground_plane.reshape((1, 4)) * stl_hom).sum(-1) > 0
+        stl_above = stl[above]
+
+        if use_gpu and torch.cuda.is_available():
+            # GPU-accelerated distance computation
+            mean_s2d = self._knn_dist_gpu(stl_above, data_in, max_dist)
+        else:
+            # CPU version (original, for exact reproduction)
+            nn_engine.fit(data_in)
+            dist_s2d, _ = nn_engine.kneighbors(stl_above, n_neighbors=1, return_distance=True)
+            mean_s2d = dist_s2d[dist_s2d < max_dist].mean()
+
+        overall = (mean_d2s + mean_s2d) / 2
+        return mean_d2s, mean_s2d, overall
+
+    def _knn_dist_gpu(
+        self,
+        query: np.ndarray,
+        target: np.ndarray,
+        max_dist: float,
+        batch_size: int = 8192,
+        target_batch_size: int = 50000,
+    ) -> float:
+        """
+        GPU-accelerated nearest neighbor distance computation.
+
+        Args:
+            query: Query points [N, 3]
+            target: Target points [M, 3]
+            max_dist: Outlier threshold
+            batch_size: Batch size for query to avoid OOM (tuned for 16GB GPU)
+            target_batch_size: Batch size for target to avoid OOM
+
+        Returns:
+            Mean distance (excluding outliers)
+        """
+        device = torch.device("cuda")
+
+        all_min_dists = []
+        n_query_batches = (len(query) + batch_size - 1) // batch_size
+        n_target_batches = (len(target) + target_batch_size - 1) // target_batch_size
+
+        # Pre-load target batches to GPU to avoid repeated transfers
+        # Memory: ~50000 pts * 3 coords * 4 bytes * n_batches
+        target_batches = []
+        for j in range(0, len(target), target_batch_size):
+            target_batch = target[j : j + target_batch_size]
+            target_t = torch.from_numpy(target_batch).float().to(device)
+            target_batches.append(target_t)
+
+        with tqdm(total=n_query_batches, desc="  GPU KNN", leave=False, ncols=100) as pbar:
+            for i in range(0, len(query), batch_size):
+                batch = query[i : i + batch_size]
+                query_t = torch.from_numpy(batch).float().to(device)
+
+                # Compute distances to all target batches
+                # Memory peak: query_batch × target_batch_size × 4 bytes
+                # = 8192 × 50000 × 4 = ~1.6 GB per cdist call
+                batch_min_dists = []
+                for target_t in target_batches:
+                    dists = torch.cdist(query_t, target_t)
+                    batch_min_dists.append(dists.min(dim=1).values)
+                    del dists  # Free immediately
+
+                # Get minimum distance across all target batches
+                min_dists = torch.stack(batch_min_dists, dim=1).min(dim=1).values
+                all_min_dists.append(min_dists.cpu().numpy())
+
+                del query_t, min_dists, batch_min_dists
+                pbar.update(1)
+
+        # Clean up target batches
+        for target_t in target_batches:
+            del target_t
+        torch.cuda.empty_cache()
+
+        all_min_dists = np.concatenate(all_min_dists)
+        return all_min_dists[all_min_dists < max_dist].mean()
+
+    def _read_ply(self, file: str) -> np.ndarray:
+        """Read point cloud from PLY file."""
+        data = PlyData.read(file)
+        vertex = data["vertex"]
+        return np.stack([vertex["x"], vertex["y"], vertex["z"]], axis=-1)
+
+    # ------------------------------
+    # Private helpers
+    # ------------------------------
+
+    def _depth_mask_path(self, scene: str, depth_idx: int) -> str:
+        """Get path to depth mask for a scene and frame."""
+        return os.path.join(
+            self.data_root, "depth_raw", "Depths", scene, f"depth_visual_{depth_idx:04d}.png"
+        )
+
+    def _prep_unposed(
+        self, pred_data: Dict, gt_data: Dict, masks: np.ndarray
+    ) -> tuple:
+        """
+        Prepare depths/intrinsics/extrinsics for recon_unposed mode.
+        
+        Applies Umeyama scale, rescales intrinsics if depth resolution differs,
+        and zeroes invalid-mask depths (nearest interpolation as in paper).
+        """
+        _, _, scale, extrinsics = align_poses_umeyama(
+            gt_data.extrinsics.copy(),
+            pred_data.extrinsics.copy(),
+            ransac=True,
+            return_aligned=True,
+            random_state=42,
+        )
+        depths = pred_data.depth * scale
+        intrinsics = pred_data.intrinsics.copy()
+
+        if depths.shape[-2:] != masks.shape[-2:]:
+            # When resizing depths to mask size, adjust intrinsics accordingly
+            sx = masks.shape[-1] / depths.shape[-1]
+            sy = masks.shape[-2] / depths.shape[-2]
+            intrinsics[:, 0:1] *= sx
+            intrinsics[:, 1:2] *= sy
+            depths = F.interpolate(
+                torch.from_numpy(depths)[None].float(),
+                size=(masks.shape[-2], masks.shape[-1]),
+                mode="nearest",
+            )[0].numpy()
+            depths[masks == False] = 0.0  # noqa: E712
+
+        return depths, intrinsics, extrinsics
+
+    def _prep_posed(
+        self, pred_data: Dict, gt_data: Dict, masks: np.ndarray
+    ) -> tuple:
+        """
+        Prepare depths/intrinsics/extrinsics for recon_posed mode.
+
+        Uses GT intrinsics/extrinsics but aligns scale via Umeyama.
+        Same mask order as other datasets: mask BEFORE scale.
+        """
+        _, _, scale, _ = align_poses_umeyama(
+            gt_data.extrinsics.copy(),
+            pred_data.extrinsics.copy(),
+            ransac=True,
+            return_aligned=True,
+            random_state=42,
+        )
+        depths = pred_data.depth.copy()
+        intrinsics = gt_data.intrinsics.copy()
+        extrinsics = gt_data.extrinsics.copy()
+
+        if depths.shape[-2:] != masks.shape[-2:]:
+            depths = F.interpolate(
+                torch.from_numpy(depths)[None].float(),
+                size=(masks.shape[-2], masks.shape[-1]),
+                mode="nearest",
+            )[0].numpy()
+
+        # Mask BEFORE scale (same as other datasets)
+        depths[masks == False] = 0.0  # noqa: E712
+        depths = depths * scale
+
+        return depths, intrinsics, extrinsics
+
+    def _build_proj_mats(
+        self, intrinsics: np.ndarray, extrinsics: np.ndarray
+    ) -> np.ndarray:
+        """Compute per-view 4x4 projection matrices from K and [R|t]."""
+        proj_mat_list = []
+        for i in range(len(intrinsics)):
+            proj_mat = np.eye(4, dtype=np.float32)
+            proj_mat[:3, :4] = np.dot(intrinsics[i], extrinsics[i][:3])
+            proj_mat_list.append(proj_mat)
+        return np.stack(proj_mat_list, axis=0)
+
+    def _fuse_consistent_points(
+        self,
+        depths_t: torch.Tensor,
+        proj_t: torch.Tensor,
+        idx: int,
+        H: int,
+        W: int,
+    ) -> np.ndarray:
+        """Fuse points consistent across multiple source views for a reference index."""
+        device, dtype = depths_t.device, depths_t.dtype
+        pc_buff = torch.zeros((3, H, W), device=device, dtype=dtype)
+        val_cnt = torch.zeros((1, H, W), device=device, dtype=dtype)
+
+        j = 0
+        batch_size = 20
+        tot_frame = depths_t.shape[0]
+        while True:
+            ref_pc, pcs, dist = self._filter_depth(
+                ref_depth=depths_t[idx : idx + 1],
+                src_depths=depths_t[j : min(j + batch_size, tot_frame)],
+                ref_proj=proj_t[idx : idx + 1],
+                src_projs=proj_t[j : min(j + batch_size, tot_frame)],
+            )
+            masks = (dist < self.dist_thresh).float()
+            masked_pc = pcs * masks
+            pc_buff += masked_pc.sum(dim=0, keepdim=False)
+            val_cnt += masks.sum(dim=0, keepdim=False)
+            j += batch_size
+            if j >= tot_frame:
+                break
+
+        final_mask = (val_cnt >= self.num_consist).squeeze(0)
+        avg_points = torch.div(pc_buff, val_cnt).permute(1, 2, 0)
+        final_pc = self._extract_points(avg_points, final_mask)
+        return final_pc
+
+    def _cap_points(self, points: np.ndarray, max_points: int) -> np.ndarray:
+        """Downsample points if exceeding max count."""
+        if len(points) <= max_points:
+            return points
+        # Use fixed seed for reproducibility
+        rng = np.random.default_rng(seed=42)
+        random_idx = rng.choice(len(points), max_points, replace=False)
+        return points[random_idx]
+

depth_anything_3/bench/datasets/dtu64.py

@@ -0,0 +1,182 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+DTU-64 Dataset implementation for POSE EVALUATION ONLY.
+
+This is a subset of DTU with 64 images per scene, specifically designed for
+camera pose estimation evaluation. It does NOT support 3D reconstruction.
+
+Note: GT depth loading is not implemented as it's not needed for pose evaluation.
+"""
+
+import glob
+import os
+from typing import Dict as TDict
+
+import numpy as np
+from addict import Dict
+
+from depth_anything_3.bench.dataset import Dataset
+from depth_anything_3.bench.registries import MONO_REGISTRY, MV_REGISTRY
+from depth_anything_3.utils.constants import (
+    DTU64_CAMERA_ROOT,
+    DTU64_EVAL_DATA_ROOT,
+    DTU64_SCENES,
+)
+
+
+@MV_REGISTRY.register(name="dtu64")
+@MONO_REGISTRY.register(name="dtu64")
+class DTU64(Dataset):
+    """
+    DTU-64 Dataset wrapper for DepthAnything3 POSE EVALUATION ONLY.
+
+    This dataset is a subset of DTU with 64 images per scene.
+    It is specifically designed for camera pose estimation evaluation
+    and does NOT support 3D reconstruction evaluation.
+
+    Dataset structure:
+        DTU/scans/
+        ├── {scene}/
+        │   └── image/          # RGB images (64 per scene)
+        └── Cameras/
+            └── {idx}_cam.txt   # Camera parameters
+
+    Supported modes:
+        - pose: Camera pose estimation evaluation
+
+    NOT supported:
+        - recon_unposed: 3D reconstruction (no GT depth available)
+        - recon_posed: 3D reconstruction (no GT depth available)
+    """
+
+    data_root = DTU64_EVAL_DATA_ROOT
+    camera_root = DTU64_CAMERA_ROOT
+    SCENES = DTU64_SCENES
+
+    def __init__(self):
+        super().__init__()
+        self._scene_cache = {}
+
+    # ------------------------------
+    # Camera file parsing
+    # ------------------------------
+
+    def read_cam_file(self, filename: str) -> tuple:
+        """
+        Read DTU camera file containing extrinsics and intrinsics.
+
+        Args:
+            filename: Path to camera text file
+
+        Returns:
+            Tuple of (intrinsics [3,3], extrinsics [4,4])
+        """
+        with open(filename) as f:
+            lines = [line.rstrip() for line in f.readlines()]
+        # extrinsics: line [1,5), 4x4 matrix
+        extrinsics = np.fromstring(" ".join(lines[1:5]), dtype=np.float32, sep=" ").reshape((4, 4))
+        # intrinsics: line [7-10), 3x3 matrix
+        intrinsics = np.fromstring(" ".join(lines[7:10]), dtype=np.float32, sep=" ").reshape((3, 3))
+        return intrinsics, extrinsics
+
+    # ------------------------------
+    # Public API
+    # ------------------------------
+
+    def get_data(self, scene: str) -> Dict:
+        """
+        Collect per-view image paths, intrinsics/extrinsics for a scene.
+
+        Args:
+            scene: Scene identifier (e.g., "scan105")
+
+        Returns:
+            Dict with:
+                - image_files: List[str] - paths to images (64 per scene)
+                - extrinsics: np.ndarray [N, 4, 4] - world-to-camera transforms
+                - intrinsics: np.ndarray [N, 3, 3] - camera intrinsics
+                - aux: Dict (empty for this dataset)
+        """
+        if scene in self._scene_cache:
+            return self._scene_cache[scene]
+
+        rgb_folder = os.path.join(self.data_root, scene, "image")
+
+        # Get all PNG files sorted
+        files = sorted(glob.glob(os.path.join(rgb_folder, "*.png")))
+
+        # Reorder: place index 33 first (reference view convention)
+        if len(files) > 33:
+            files = [files[33]] + files[:33] + files[34:]
+
+        out = Dict({
+            "image_files": [],
+            "extrinsics": [],
+            "intrinsics": [],
+            "aux": Dict({}),
+        })
+
+        for rgb_file in files:
+            basename = os.path.basename(rgb_file)
+            # File naming: "00000033.png" -> cam_idx = 33
+            file_idx = basename.split(".")[0]
+            cam_idx = int(file_idx)
+
+            # Camera file path
+            cam_file = os.path.join(self.camera_root, f"{cam_idx:0>8}_cam.txt")
+
+            if not os.path.exists(cam_file):
+                print(f"[DTU-64] Warning: Camera file not found: {cam_file}")
+                continue
+
+            intrinsics, extrinsics = self.read_cam_file(cam_file)
+
+            out.image_files.append(rgb_file)
+            out.extrinsics.append(extrinsics)
+            out.intrinsics.append(intrinsics)
+
+        out.extrinsics = np.asarray(out.extrinsics, dtype=np.float32)
+        out.intrinsics = np.asarray(out.intrinsics, dtype=np.float32)
+
+        print(f"[DTU-64] {scene}: {len(out.image_files)} images (pose evaluation only)")
+
+        self._scene_cache[scene] = out
+        return out
+
+    def eval3d(self, scene: str, fuse_path: str) -> TDict[str, float]:
+        """
+        NOT SUPPORTED for DTU-64.
+
+        DTU-64 is only for pose evaluation, not 3D reconstruction.
+        """
+        raise NotImplementedError(
+            "DTU-64 dataset is for POSE EVALUATION ONLY. "
+            "3D reconstruction evaluation is not supported. "
+            "Use the standard 'dtu' dataset for 3D reconstruction evaluation."
+        )
+
+    def fuse3d(self, scene: str, result_path: str, fuse_path: str, mode: str) -> None:
+        """
+        NOT SUPPORTED for DTU-64.
+
+        DTU-64 is only for pose evaluation, not 3D reconstruction.
+        """
+        raise NotImplementedError(
+            "DTU-64 dataset is for POSE EVALUATION ONLY. "
+            "3D reconstruction (fuse3d) is not supported. "
+            "Use the standard 'dtu' dataset for 3D reconstruction."
+        )
+

depth_anything_3/bench/datasets/eth3d.py

@@ -0,0 +1,594 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+ETH3D Benchmark dataset implementation.
+
+ETH3D is a multi-view stereo benchmark with high-resolution images and
+accurate ground truth geometry from laser scanning.
+Reference: https://www.eth3d.net/
+
+Evaluation metrics:
+- 3D reconstruction: Accuracy, Completeness, F-score
+- Camera pose estimation: AUC metrics
+"""
+
+import glob
+import os
+from typing import Dict as TDict, List, Optional
+
+import cv2
+import numpy as np
+import open3d as o3d
+import torch
+import torch.nn.functional as F
+from addict import Dict
+from PIL import Image
+
+from depth_anything_3.bench.dataset import Dataset, _wait_for_file_ready
+from depth_anything_3.bench.registries import MONO_REGISTRY, MV_REGISTRY
+from depth_anything_3.bench.utils import (
+    create_tsdf_volume,
+    evaluate_3d_reconstruction,
+    fuse_depth_to_tsdf,
+    quat2rotmat,
+    sample_points_from_mesh,
+)
+from depth_anything_3.utils.constants import (
+    ETH3D_DOWN_SAMPLE,
+    ETH3D_EVAL_DATA_ROOT,
+    ETH3D_EVAL_THRESHOLD,
+    ETH3D_FILTER_KEYS,
+    ETH3D_MAX_DEPTH,
+    ETH3D_SAMPLING_NUMBER,
+    ETH3D_SCENES,
+    ETH3D_SDF_TRUNC,
+    ETH3D_VOXEL_LENGTH,
+)
+from depth_anything_3.utils.pose_align import align_poses_umeyama
+
+
+@MV_REGISTRY.register(name="eth3d")
+@MONO_REGISTRY.register(name="eth3d")
+class ETH3D(Dataset):
+    """
+    ETH3D Benchmark dataset wrapper for DepthAnything3 evaluation.
+
+    Supports:
+        - Camera pose estimation evaluation (AUC metrics)
+        - 3D reconstruction evaluation (Accuracy, Completeness, F-score)
+        - TSDF-based point cloud fusion
+
+    Dataset structure:
+        eth3d/multiview/
+        ├── scene_name/
+        │   ├── images/                    # RGB images
+        │   ├── dslr_calibration_jpg/
+        │   │   ├── cameras.txt            # Camera intrinsics
+        │   │   └── images.txt             # Camera poses
+        │   ├── combined_mesh.ply          # Ground truth mesh
+        │   └── ground_truth_depth/        # GT depth maps (optional)
+    """
+
+    data_root = ETH3D_EVAL_DATA_ROOT
+    SCENES = ETH3D_SCENES
+
+    # Evaluation hyperparameters from constants
+    max_depth = ETH3D_MAX_DEPTH
+    sampling_number = ETH3D_SAMPLING_NUMBER
+    voxel_length = ETH3D_VOXEL_LENGTH
+    sdf_trunc = ETH3D_SDF_TRUNC
+    eval_threshold = ETH3D_EVAL_THRESHOLD
+    down_sample = ETH3D_DOWN_SAMPLE
+
+    def __init__(self):
+        super().__init__()
+        # Pre-load scene data for efficiency
+        self._scene_cache = {}
+
+    # ------------------------------
+    # Camera file parsing
+    # ------------------------------
+
+    def _parse_cameras_txt(self, filepath: str) -> dict:
+        """
+        Parse COLMAP-style cameras.txt file.
+
+        Returns:
+            Dict mapping camera_id to intrinsic parameters
+        """
+        camera_dict = {}
+        with open(filepath) as f:
+            lines = f.readlines()
+            for line in lines[3:]:  # Skip header
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+                parts = line.split()
+                if len(parts) < 8:
+                    continue
+                cam_id = parts[0]
+                # Format: ID, MODEL, WIDTH, HEIGHT, fx, fy, cx, cy, [distortion params...]
+                camera_dict[cam_id] = {
+                    "width": float(parts[2]),
+                    "height": float(parts[3]),
+                    "fx": float(parts[4]),
+                    "fy": float(parts[5]),
+                    "cx": float(parts[6]),
+                    "cy": float(parts[7]),
+                }
+        return camera_dict
+
+    def _parse_images_txt(self, filepath: str) -> dict:
+        """
+        Parse COLMAP-style images.txt file.
+
+        Returns:
+            Dict mapping image path to pose parameters
+        """
+        pose_dict = {}
+        with open(filepath) as f:
+            lines = f.readlines()
+            for idx, line in enumerate(lines[4:]):  # Skip header
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+                # Every other line contains pose info
+                if idx % 2 == 0:
+                    parts = line.split()
+                    if len(parts) < 10:
+                        continue
+                    # Format: IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME
+                    image_id = parts[0]
+                    qw, qx, qy, qz = float(parts[1]), float(parts[2]), float(parts[3]), float(parts[4])
+                    tx, ty, tz = float(parts[5]), float(parts[6]), float(parts[7])
+                    camera_id = parts[8]
+                    name = parts[9]
+                    pose_dict[name] = {
+                        "image_id": image_id,
+                        "quat": [qw, qx, qy, qz],
+                        "trans": [tx, ty, tz],
+                        "camera_id": camera_id,
+                    }
+        return pose_dict
+
+    def _should_filter_image(self, scene: str, image_name: str) -> bool:
+        """Check if image should be filtered out based on known problematic views."""
+        filter_keys = ETH3D_FILTER_KEYS.get(scene, [])
+        for key in filter_keys:
+            if image_name.endswith(key):
+                return True
+        return False
+
+    # ------------------------------
+    # Public API
+    # ------------------------------
+
+    def get_data(self, scene: str) -> Dict:
+        """
+        Collect per-view image paths, intrinsics/extrinsics for a scene.
+
+        Args:
+            scene: Scene identifier (e.g., "courtyard")
+
+        Returns:
+            Dict with:
+                - image_files: List[str] - paths to images
+                - extrinsics: np.ndarray [N, 4, 4] - world-to-camera transforms
+                - intrinsics: np.ndarray [N, 3, 3] - camera intrinsics
+                - aux: Dict with gt_mesh_path
+        """
+        # Check cache
+        if scene in self._scene_cache:
+            return self._scene_cache[scene]
+
+        scene_dir = os.path.join(self.data_root, scene)
+
+        # Parse camera files
+        cameras_file = os.path.join(scene_dir, "dslr_calibration_jpg", "cameras.txt")
+        images_file = os.path.join(scene_dir, "dslr_calibration_jpg", "images.txt")
+        camera_dict = self._parse_cameras_txt(cameras_file)
+        pose_dict = self._parse_images_txt(images_file)
+
+        # Ground truth mesh path
+        gt_mesh_path = os.path.join(scene_dir, "combined_mesh.ply")
+
+        out = Dict({
+            "image_files": [],
+            "extrinsics": [],
+            "intrinsics": [],
+            "aux": Dict({
+                "gt_mesh_path": gt_mesh_path,
+                "heights": [],
+                "widths": [],
+            }),
+        })
+
+        # Process each image (preserve original order from images.txt)
+        filtered_count = 0
+        for image_name, pose_info in pose_dict.items():
+            # Filter problematic views
+            if self._should_filter_image(scene, image_name):
+                filtered_count += 1
+                continue
+
+            image_path = os.path.join(scene_dir, "images", image_name)
+            if not os.path.exists(image_path):
+                continue
+
+            cam_info = camera_dict.get(pose_info["camera_id"])
+            if cam_info is None:
+                continue
+
+            # Build intrinsics matrix
+            ixt = np.array([
+                [cam_info["fx"], 0, cam_info["cx"]],
+                [0, cam_info["fy"], cam_info["cy"]],
+                [0, 0, 1],
+            ], dtype=np.float32)
+
+            # Build extrinsics matrix (world-to-camera)
+            # COLMAP format: world point -> camera point
+            rot = quat2rotmat(pose_info["quat"])
+            ext = np.eye(4, dtype=np.float32)
+            ext[:3, :3] = rot
+            ext[:3, 3] = pose_info["trans"]
+
+            out.image_files.append(image_path)
+            out.extrinsics.append(ext)
+            out.intrinsics.append(ixt)
+            out.aux.heights.append(cam_info["height"])
+            out.aux.widths.append(cam_info["width"])
+
+        out.extrinsics = np.asarray(out.extrinsics, dtype=np.float32)
+        out.intrinsics = np.asarray(out.intrinsics, dtype=np.float32)
+
+        # Print scene info
+        total_images = len(pose_dict)
+        used_images = len(out.image_files)
+        print(f"[ETH3D] {scene}: {used_images}/{total_images} images "
+              f"(filtered {filtered_count}, missing {total_images - used_images - filtered_count})")
+        
+        if used_images < 3:
+            print(f"[ETH3D] ⚠️  WARNING: {scene} has only {used_images} images - evaluation may fail!")
+
+        # Cache result
+        self._scene_cache[scene] = out
+        return out
+
+    def eval3d(self, scene: str, fuse_path: str) -> TDict[str, float]:
+        """
+        Evaluate fused point cloud against ETH3D ground truth mesh.
+
+        Args:
+            scene: Scene identifier
+            fuse_path: Path to fused point cloud (.ply)
+
+        Returns:
+            Dict with metrics: acc, comp, overall, precision, recall, fscore
+        """
+        gt_data = self.get_data(scene)
+        gt_mesh_path = gt_data.aux.gt_mesh_path
+
+        # Load and sample ground truth mesh
+        gt_mesh = o3d.io.read_triangle_mesh(gt_mesh_path)
+        gt_pcd = sample_points_from_mesh(gt_mesh, self.sampling_number)
+
+        # Load predicted point cloud
+        pred_pcd = o3d.io.read_point_cloud(fuse_path)
+
+        # Evaluate using shared utility function
+        metrics = evaluate_3d_reconstruction(
+            pred_pcd,
+            gt_pcd,
+            threshold=self.eval_threshold,
+            down_sample=self.down_sample,
+        )
+
+        return metrics
+
+    def _load_gt_meta(self, result_path: str) -> Dict:
+        """
+        Load saved GT meta (extrinsics, intrinsics, image_files) for fusion.
+
+        This is needed when frames are sampled, so fuse3d uses the correct
+        (sampled) GT instead of full dataset GT.
+
+        Args:
+            result_path: Path to npz file (used to derive gt_meta.npz path)
+
+        Returns:
+            Dict with GT data, or None if gt_meta.npz doesn't exist
+        """
+        # gt_meta.npz is in the same exports/ directory as results.npz
+        export_dir = os.path.dirname(result_path)  # exports/mini_npz/
+        gt_meta_path = os.path.join(os.path.dirname(export_dir), "gt_meta.npz")
+
+        if os.path.exists(gt_meta_path):
+            data = np.load(gt_meta_path, allow_pickle=True)
+            return Dict({
+                "extrinsics": data["extrinsics"],
+                "intrinsics": data["intrinsics"],
+                "image_files": data["image_files"] if "image_files" in data else None,
+            })
+        return None
+
+    def fuse3d(self, scene: str, result_path: str, fuse_path: str, mode: str) -> None:
+        """
+        Fuse per-view depths into a point cloud using TSDF fusion.
+
+        Pipeline:
+        1. Load original images (keep original size)
+        2. Resize depth to original image size (nearest interpolation)
+        3. Adjust intrinsics to original image size
+        4. Apply scale alignment and mask invalid depths
+        5. TSDF fusion
+
+        Args:
+            scene: Scene identifier
+            result_path: Path to npz file with predicted depths/poses
+            fuse_path: Output path for fused point cloud (.ply)
+            mode: "recon_unposed" or "recon_posed"
+        """
+        # Try to load saved GT meta (handles frame sampling)
+        gt_meta = self._load_gt_meta(result_path)
+        if gt_meta is not None:
+            gt_data = gt_meta
+        else:
+            gt_data = self.get_data(scene)
+        _wait_for_file_ready(result_path)
+        pred_data = Dict({k: v for k, v in np.load(result_path).items()})
+
+        # Load original images (keep original size)
+        images = []
+        orig_sizes = []  # (H, W) for each image
+        for img_path in gt_data.image_files:
+            img = cv2.imread(img_path)
+            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+            images.append(img)
+            orig_sizes.append((img.shape[0], img.shape[1]))
+
+        # Prepare depths, intrinsics, extrinsics with resize to original size
+        if mode == "recon_unposed":
+            depths, intrinsics, extrinsics = self._prep_unposed(
+                pred_data, gt_data, orig_sizes, scene=scene
+            )
+        elif mode == "recon_posed":
+            depths, intrinsics, extrinsics = self._prep_posed(
+                pred_data, gt_data, orig_sizes, scene=scene
+            )
+        else:
+            raise ValueError(f"Invalid mode: {mode}")
+
+        images = np.stack(images, axis=0)
+
+        # Create TSDF volume and fuse
+        volume = create_tsdf_volume(
+            voxel_length=self.voxel_length,
+            sdf_trunc=self.sdf_trunc,
+        )
+        mesh = fuse_depth_to_tsdf(
+            volume, depths, images, intrinsics, extrinsics, max_depth=self.max_depth
+        )
+
+        # Sample points from mesh
+        pcd = sample_points_from_mesh(mesh, self.sampling_number)
+
+        # Save point cloud
+        os.makedirs(os.path.dirname(fuse_path), exist_ok=True)
+        o3d.io.write_point_cloud(fuse_path, pcd)
+
+    # ------------------------------
+    # Private helpers
+    # ------------------------------
+
+    def _prep_unposed(
+        self, pred_data: Dict, gt_data: Dict, orig_sizes: list, scene: str = None
+    ) -> tuple:
+        """
+        Prepare depths/intrinsics/extrinsics for recon_unposed mode.
+
+        Pipeline:
+        1. Umeyama scale alignment
+        2. Load GT mask for each frame
+        3. Resize depth to original image size (nearest)
+        4. Apply GT mask BEFORE scale
+        5. Apply scale
+        6. Adjust intrinsics to original image size
+        """
+        # Scale alignment with fixed random_state for reproducibility
+        _, _, scale, extrinsics = align_poses_umeyama(
+            gt_data.extrinsics.copy(),
+            pred_data.extrinsics.copy(),
+            return_aligned=True,
+            ransac=True,
+            random_state=42,
+        )
+
+        # Get model output size
+        model_h, model_w = pred_data.depth.shape[1], pred_data.depth.shape[2]
+
+        # Process each frame
+        depths_out = []
+        intrinsics_out = []
+        for i in range(len(pred_data.depth)):
+            orig_h, orig_w = orig_sizes[i]
+            image_name = os.path.basename(gt_data.image_files[i])
+
+            # Resize depth to original image size (nearest interpolation)
+            depth = cv2.resize(
+                pred_data.depth[i],
+                (orig_w, orig_h),
+                interpolation=cv2.INTER_NEAREST,
+            )
+
+            # Load GT mask (apply BEFORE scale)
+            gt_zero_mask = None
+            if scene is not None:
+                gt_zero_mask = self._load_gt_mask(scene, image_name, (orig_h, orig_w))
+
+            # Mask invalid depths BEFORE scale
+            depth = self._mask_invalid_depth(depth, gt_zero_mask)
+
+            # Apply scale AFTER mask
+            depth = depth * scale
+
+            # Adjust intrinsics to original image size
+            h_ratio = orig_h / model_h
+            w_ratio = orig_w / model_w
+            ixt = pred_data.intrinsics[i].copy()
+            ixt[0, :] *= w_ratio  # fx, 0, cx
+            ixt[1, :] *= h_ratio  # 0, fy, cy
+
+            depths_out.append(depth)
+            intrinsics_out.append(ixt)
+
+        return np.stack(depths_out), np.stack(intrinsics_out), extrinsics
+
+    def _prep_posed(
+        self, pred_data: Dict, gt_data: Dict, orig_sizes: list, scene: str = None
+    ) -> tuple:
+        """
+        Prepare depths/intrinsics/extrinsics for recon_posed mode.
+
+        Uses GT intrinsics/extrinsics but aligns depth scale via Umeyama.
+        Depth is resized to original image size.
+        """
+        # Scale alignment with fixed random_state for reproducibility
+        _, _, scale, _ = align_poses_umeyama(
+            gt_data.extrinsics.copy(),
+            pred_data.extrinsics.copy(),
+            return_aligned=True,
+            ransac=True,
+            random_state=42,
+        )
+
+        # Process each frame
+        depths_out = []
+        for i in range(len(pred_data.depth)):
+            orig_h, orig_w = orig_sizes[i]
+            image_name = os.path.basename(gt_data.image_files[i])
+
+            # Resize depth to original image size (nearest interpolation)
+            depth = cv2.resize(
+                pred_data.depth[i],
+                (orig_w, orig_h),
+                interpolation=cv2.INTER_NEAREST,
+            )
+
+            # Load GT mask (apply BEFORE scale)
+            gt_zero_mask = None
+            if scene is not None:
+                gt_zero_mask = self._load_gt_mask(scene, image_name, (orig_h, orig_w))
+
+            # Mask invalid depths BEFORE scale
+            depth = self._mask_invalid_depth(depth, gt_zero_mask)
+
+            # Apply scale AFTER mask
+            depth = depth * scale
+
+            depths_out.append(depth)
+
+        # Use GT intrinsics and extrinsics (already at original image size)
+        return np.stack(depths_out), gt_data.intrinsics.copy(), gt_data.extrinsics.copy()
+
+    def _load_gt_mask(self, scene: str, image_name: str, shape: tuple) -> np.ndarray:
+        """
+        Load GT mask for masking invalid regions.
+
+        GT mask marks occluded or invalid regions that should be excluded
+        from depth fusion and evaluation.
+
+        Args:
+            scene: Scene identifier
+            image_name: Image filename (e.g., "DSC_0307.JPG")
+            shape: (height, width) of the image
+
+        Returns:
+            Boolean mask where True = valid region to keep
+        """
+        h, w = shape
+
+        # GT mask file path
+        gt_mask_path = os.path.join(
+            self.data_root, scene, "masks_for_images", "dslr_images",
+            image_name.replace(".JPG", ".png")
+        )
+
+        # GT depth file path (used to determine valid depth regions)
+        gt_depth_path = os.path.join(
+            self.data_root, scene, "ground_truth_depth", "dslr_images", image_name
+        )
+
+        # Load GT depth
+        if os.path.exists(gt_depth_path):
+            gt_depth = np.fromfile(gt_depth_path, dtype=np.float32).reshape(h, w)
+        else:
+            gt_depth = np.ones((h, w), dtype=np.float32)
+
+        # Load GT mask
+        if os.path.exists(gt_mask_path):
+            gt_mask = cv2.imread(gt_mask_path, cv2.IMREAD_GRAYSCALE)
+            gt_mask = np.asarray(gt_mask)
+        else:
+            gt_mask = np.zeros((h, w), dtype=np.uint8)
+
+        # Compute zero_mask
+        # gt_mask == 1 means occluded/invalid region
+        invalid_mask_from_gt = gt_mask == 1
+        gt_depth_copy = gt_depth.copy()
+        gt_depth_copy[gt_mask == 1] = 0
+
+        invalid_mask_from_gt_depth = np.logical_or(gt_depth_copy == 0, gt_depth_copy == np.inf)
+
+        # zero_mask: valid region that should be kept
+        zero_mask = np.logical_and(
+            np.logical_not(invalid_mask_from_gt),
+            np.logical_not(invalid_mask_from_gt_depth)
+        )
+
+        return zero_mask
+
+    def _mask_invalid_depth(
+        self, depth: np.ndarray, gt_zero_mask: np.ndarray = None
+    ) -> np.ndarray:
+        """
+        Mask invalid depth values by setting them to 0.
+
+        Logic:
+        1. Apply GT mask (if provided) - marks occluded/invalid regions
+        2. Mask pred invalid values (nan, inf)
+
+        Args:
+            depth: Depth map to mask
+            gt_zero_mask: Optional GT mask (True = valid region)
+
+        Returns:
+            Masked depth map with invalid regions set to 0
+        """
+        depth = depth.copy()
+
+        # Apply GT mask first (before scale)
+        if gt_zero_mask is not None:
+            # Also mask out invalid pred depth
+            pred_invalid = np.isnan(depth) | np.isinf(depth)
+            combined_mask = np.logical_and(gt_zero_mask, np.logical_not(pred_invalid))
+            depth = depth * combined_mask.astype(np.float32)
+        else:
+            # Fallback: only mask pred invalid values
+            invalid_mask = np.isnan(depth) | np.isinf(depth) | (depth <= 0)
+            depth[invalid_mask] = 0.0
+
+        return depth
+

depth_anything_3/bench/datasets/hiroom.py

@@ -0,0 +1,440 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+HiRoom Dataset implementation.
+
+HiRoom is an indoor RGB-D dataset containing ground truth camera poses,
+depth maps, and fused point clouds.
+
+Evaluation metrics:
+- 3D reconstruction: Accuracy, Completeness, F-score
+- Camera pose estimation: AUC metrics
+"""
+
+import os
+from typing import Dict as TDict, List
+
+import cv2
+import numpy as np
+import open3d as o3d
+from addict import Dict
+
+from depth_anything_3.bench.dataset import Dataset, _wait_for_file_ready
+from depth_anything_3.bench.registries import MONO_REGISTRY, MV_REGISTRY
+from depth_anything_3.bench.utils import (
+    create_tsdf_volume,
+    evaluate_3d_reconstruction,
+    fuse_depth_to_tsdf,
+    sample_points_from_mesh,
+)
+from depth_anything_3.utils.constants import (
+    HIROOM_DOWN_SAMPLE,
+    HIROOM_EVAL_DATA_ROOT,
+    HIROOM_EVAL_THRESHOLD,
+    HIROOM_GT_ROOT_PATH,
+    HIROOM_MAX_DEPTH,
+    HIROOM_SAMPLING_NUMBER,
+    HIROOM_SCENE_LIST_PATH,
+    HIROOM_SDF_TRUNC,
+    HIROOM_VOXEL_LENGTH,
+)
+from depth_anything_3.utils.pose_align import align_poses_umeyama
+
+
+def _load_scene_list() -> List[str]:
+    """Load scene list from file."""
+    if os.path.exists(HIROOM_SCENE_LIST_PATH):
+        with open(HIROOM_SCENE_LIST_PATH, "r") as f:
+            return f.read().splitlines()
+    return []
+
+
+@MV_REGISTRY.register(name="hiroom")
+@MONO_REGISTRY.register(name="hiroom")
+class HiRoomDataset(Dataset):
+    """
+    HiRoom Dataset wrapper for DepthAnything3 evaluation.
+
+    Supports:
+        - Camera pose estimation evaluation (AUC metrics)
+        - 3D reconstruction evaluation (Accuracy, Completeness, F-score)
+        - TSDF-based point cloud fusion
+
+    Dataset structure:
+        HiRoom/
+        ├── {scene_path}/
+        │   ├── image/           # RGB images
+        │   ├── depth/           # GT depth maps
+        │   ├── pose/            # Camera poses (.npy)
+        │   ├── cam_K.npy        # Camera intrinsics
+        │   └── aliasing_mask/   # Aliasing masks
+        
+        fused_pcd/
+        └── {scene_name}.ply     # Ground truth fused point cloud
+    """
+
+    data_root = HIROOM_EVAL_DATA_ROOT
+    gt_root_path = HIROOM_GT_ROOT_PATH
+    SCENES = _load_scene_list()
+
+    # Evaluation hyperparameters from constants
+    max_depth = HIROOM_MAX_DEPTH
+    sampling_number = HIROOM_SAMPLING_NUMBER
+    voxel_length = HIROOM_VOXEL_LENGTH
+    sdf_trunc = HIROOM_SDF_TRUNC
+    eval_threshold = HIROOM_EVAL_THRESHOLD
+    down_sample = HIROOM_DOWN_SAMPLE
+
+    def __init__(self):
+        super().__init__()
+        self._scene_cache = {}
+
+    # ------------------------------
+    # Public API
+    # ------------------------------
+
+    def get_data(self, scene: str) -> Dict:
+        """
+        Collect per-view image paths, intrinsics/extrinsics for a scene.
+
+        Args:
+            scene: Scene path (e.g., "xxx/yyy/zzz")
+
+        Returns:
+            Dict with:
+                - image_files: List[str] - paths to images
+                - extrinsics: np.ndarray [N, 4, 4] - world-to-camera transforms
+                - intrinsics: np.ndarray [N, 3, 3] - camera intrinsics
+                - aux: Dict with gt_pcd_path, gt_depth_files, aliasing_mask_files
+        """
+        if scene in self._scene_cache:
+            return self._scene_cache[scene]
+
+        scene_dir = os.path.join(self.data_root, scene)
+        image_dir = os.path.join(scene_dir, "image")
+
+        # Get scene name for GT point cloud
+        scene_name = "-".join(scene.split("/")[-3:])
+        gt_pcd_path = os.path.join(self.gt_root_path, f"{scene_name}.ply")
+
+        # Load shared camera intrinsics
+        intrin_path = os.path.join(scene_dir, "cam_K.npy")
+        ixt_shared = np.load(intrin_path).astype(np.float32)
+
+        # Get all image names sorted
+        image_names = sorted(os.listdir(image_dir))
+
+        out = Dict({
+            "image_files": [],
+            "extrinsics": [],
+            "intrinsics": [],
+            "aux": Dict({
+                "gt_pcd_path": gt_pcd_path,
+                "gt_depth_files": [],
+                "aliasing_mask_files": [],
+            }),
+        })
+
+        for img_name in image_names:
+            img_path = os.path.join(image_dir, img_name)
+            frame_name = img_name.split(".")[0]
+
+            # Depth and pose paths
+            depth_path = os.path.join(scene_dir, "depth", f"{frame_name}.png")
+            pose_path = os.path.join(scene_dir, "pose", f"{frame_name}.npy")
+            aliasing_mask_path = os.path.join(scene_dir, "aliasing_mask", f"{frame_name}.png")
+
+            if not os.path.exists(pose_path):
+                continue
+
+            # Load extrinsics (world-to-camera)
+            ext = np.load(pose_path).astype(np.float32)
+
+            out.image_files.append(img_path)
+            out.extrinsics.append(ext)
+            out.intrinsics.append(ixt_shared.copy())
+            out.aux.gt_depth_files.append(depth_path)
+            out.aux.aliasing_mask_files.append(aliasing_mask_path)
+
+        out.extrinsics = np.asarray(out.extrinsics, dtype=np.float32)
+        out.intrinsics = np.asarray(out.intrinsics, dtype=np.float32)
+
+        print(f"[HiRoom] {scene}: {len(out.image_files)} images")
+
+        self._scene_cache[scene] = out
+        return out
+
+    def eval3d(self, scene: str, fuse_path: str) -> TDict[str, float]:
+        """
+        Evaluate fused point cloud against HiRoom ground truth point cloud.
+
+        Args:
+            scene: Scene identifier
+            fuse_path: Path to fused point cloud (.ply)
+
+        Returns:
+            Dict with metrics: acc, comp, overall, precision, recall, fscore
+        """
+        gt_data = self.get_data(scene)
+        gt_pcd_path = gt_data.aux.gt_pcd_path
+
+        # Load ground truth point cloud
+        gt_pcd = o3d.io.read_point_cloud(gt_pcd_path)
+
+        # Load predicted point cloud
+        pred_pcd = o3d.io.read_point_cloud(fuse_path)
+
+        # Evaluate using shared utility function
+        metrics = evaluate_3d_reconstruction(
+            pred_pcd,
+            gt_pcd,
+            threshold=self.eval_threshold,
+            down_sample=self.down_sample,
+        )
+
+        return metrics
+
+    def _load_gt_meta(self, result_path: str) -> Dict:
+        """Load saved GT meta for fusion."""
+        export_dir = os.path.dirname(result_path)
+        gt_meta_path = os.path.join(os.path.dirname(export_dir), "gt_meta.npz")
+
+        if os.path.exists(gt_meta_path):
+            data = np.load(gt_meta_path, allow_pickle=True)
+            image_files = list(data["image_files"])
+            return Dict({
+                "extrinsics": data["extrinsics"],
+                "intrinsics": data["intrinsics"],
+                "image_files": image_files,
+            })
+        return None
+
+    def fuse3d(self, scene: str, result_path: str, fuse_path: str, mode: str) -> None:
+        """
+        Fuse per-view depths into a point cloud using TSDF fusion.
+
+        Args:
+            scene: Scene identifier
+            result_path: Path to npz file with predicted depths/poses
+            fuse_path: Output path for fused point cloud (.ply)
+            mode: "recon_unposed" or "recon_posed"
+        """
+        # Get full GT data
+        full_gt_data = self.get_data(scene)
+
+        # Try to load saved GT meta (handles frame sampling)
+        gt_meta = self._load_gt_meta(result_path)
+        if gt_meta is not None:
+            gt_data = gt_meta
+            image_indices = [
+                full_gt_data.image_files.index(f)
+                for f in gt_data.image_files
+                if f in full_gt_data.image_files
+            ]
+        else:
+            gt_data = full_gt_data
+            image_indices = list(range(len(full_gt_data.image_files)))
+
+        _wait_for_file_ready(result_path)
+        pred_data = Dict({k: v for k, v in np.load(result_path).items()})
+
+        # Load images
+        images = []
+        orig_sizes = []
+        for img_idx in image_indices:
+            img_path = full_gt_data.image_files[img_idx]
+            img = cv2.imread(img_path)
+            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+            images.append(img)
+            orig_sizes.append((img.shape[0], img.shape[1]))
+
+        images = np.stack(images, axis=0)
+
+        # Prepare depths, intrinsics, extrinsics
+        if mode == "recon_unposed":
+            depths, intrinsics, extrinsics = self._prep_unposed(
+                pred_data, gt_data, full_gt_data, image_indices, orig_sizes, scene=scene
+            )
+        elif mode == "recon_posed":
+            depths, intrinsics, extrinsics = self._prep_posed(
+                pred_data, gt_data, full_gt_data, image_indices, orig_sizes, scene=scene
+            )
+        else:
+            raise ValueError(f"Invalid mode: {mode}")
+
+        # Create TSDF volume and fuse
+        volume = create_tsdf_volume(
+            voxel_length=self.voxel_length,
+            sdf_trunc=self.sdf_trunc,
+        )
+        mesh = fuse_depth_to_tsdf(
+            volume, depths, images, intrinsics, extrinsics, max_depth=self.max_depth
+        )
+
+        # Sample points from mesh
+        pcd = sample_points_from_mesh(mesh, self.sampling_number)
+
+        # Save point cloud
+        os.makedirs(os.path.dirname(fuse_path), exist_ok=True)
+        o3d.io.write_point_cloud(fuse_path, pcd)
+
+    # ------------------------------
+    # Private helpers
+    # ------------------------------
+
+    def _prep_unposed(
+        self, pred_data: Dict, gt_data: Dict, full_gt_data: Dict,
+        image_indices: list, orig_sizes: list, scene: str = None
+    ) -> tuple:
+        """Prepare depths/intrinsics/extrinsics for recon_unposed mode."""
+        # Scale alignment with fixed random_state for reproducibility
+        _, _, scale, extrinsics = align_poses_umeyama(
+            gt_data.extrinsics.copy(),
+            pred_data.extrinsics.copy(),
+            return_aligned=True,
+            ransac=True,
+            random_state=42,
+        )
+
+        model_h, model_w = pred_data.depth.shape[1], pred_data.depth.shape[2]
+
+        depths_out = []
+        intrinsics_out = []
+        for i in range(len(pred_data.depth)):
+            orig_h, orig_w = orig_sizes[i]
+            img_idx = image_indices[i]
+
+            # Resize depth to original image size
+            depth = cv2.resize(
+                pred_data.depth[i],
+                (orig_w, orig_h),
+                interpolation=cv2.INTER_NEAREST,
+            )
+
+            # Load GT mask
+            gt_zero_mask = self._load_gt_mask(
+                full_gt_data.aux.gt_depth_files[img_idx],
+                full_gt_data.aux.aliasing_mask_files[img_idx],
+            )
+
+            # Mask invalid depths BEFORE scale
+            depth = self._mask_invalid_depth(depth, gt_zero_mask)
+
+            # Apply scale AFTER mask
+            depth = depth * scale
+
+            # Adjust intrinsics to original image size
+            h_ratio = orig_h / model_h
+            w_ratio = orig_w / model_w
+            ixt = pred_data.intrinsics[i].copy()
+            ixt[0, :] *= w_ratio
+            ixt[1, :] *= h_ratio
+
+            depths_out.append(depth)
+            intrinsics_out.append(ixt)
+
+        return np.stack(depths_out), np.stack(intrinsics_out), extrinsics
+
+    def _prep_posed(
+        self, pred_data: Dict, gt_data: Dict, full_gt_data: Dict,
+        image_indices: list, orig_sizes: list, scene: str = None
+    ) -> tuple:
+        """Prepare depths/intrinsics/extrinsics for recon_posed mode."""
+        # Scale alignment
+        _, _, scale, _ = align_poses_umeyama(
+            gt_data.extrinsics.copy(),
+            pred_data.extrinsics.copy(),
+            return_aligned=True,
+            ransac=True,
+            random_state=42,
+        )
+
+        depths_out = []
+        for i in range(len(pred_data.depth)):
+            orig_h, orig_w = orig_sizes[i]
+            img_idx = image_indices[i]
+
+            # Resize depth to original image size
+            depth = cv2.resize(
+                pred_data.depth[i],
+                (orig_w, orig_h),
+                interpolation=cv2.INTER_NEAREST,
+            )
+
+            # Load GT mask
+            gt_zero_mask = self._load_gt_mask(
+                full_gt_data.aux.gt_depth_files[img_idx],
+                full_gt_data.aux.aliasing_mask_files[img_idx],
+            )
+
+            # Mask invalid depths BEFORE scale
+            depth = self._mask_invalid_depth(depth, gt_zero_mask)
+
+            # Apply scale AFTER mask
+            depth = depth * scale
+
+            depths_out.append(depth)
+
+        # Use GT intrinsics and extrinsics
+        gt_intrinsics = np.stack([full_gt_data.intrinsics[idx] for idx in image_indices])
+        gt_extrinsics = np.stack([full_gt_data.extrinsics[idx] for idx in image_indices])
+
+        return np.stack(depths_out), gt_intrinsics, gt_extrinsics
+
+    def _load_gt_mask(self, gt_depth_path: str, aliasing_mask_path: str) -> np.ndarray:
+        """
+        Load GT depth and aliasing mask to create valid mask.
+
+        For HiRoom:
+        - GT depth is stored as 16-bit PNG, scaled to 100m range
+        - Aliasing mask marks regions to exclude
+
+        Returns:
+            Boolean mask where True = valid region to keep
+        """
+        # Load GT depth
+        if os.path.exists(gt_depth_path):
+            gt_depth = cv2.imread(gt_depth_path, -1) / 65535.0 * 100.0
+        else:
+            return None
+
+        # Load aliasing mask
+        aliasing_mask = None
+        if os.path.exists(aliasing_mask_path):
+            aliasing_mask = cv2.imread(aliasing_mask_path, -1) > 0
+
+        # Valid mask: depth > 0 and not in aliasing region
+        valid_mask = gt_depth > 0
+        if aliasing_mask is not None:
+            valid_mask = np.logical_and(valid_mask, np.logical_not(aliasing_mask))
+
+        return valid_mask
+
+    def _mask_invalid_depth(
+        self, depth: np.ndarray, gt_zero_mask: np.ndarray = None
+    ) -> np.ndarray:
+        """Mask invalid depth values by setting them to 0."""
+        depth = depth.copy()
+
+        if gt_zero_mask is not None:
+            pred_invalid = np.isnan(depth) | np.isinf(depth)
+            combined_mask = np.logical_and(gt_zero_mask, np.logical_not(pred_invalid))
+            depth = depth * combined_mask.astype(np.float32)
+        else:
+            invalid_mask = np.isnan(depth) | np.isinf(depth) | (depth <= 0)
+            depth[invalid_mask] = 0.0
+
+        return depth
+

depth_anything_3/bench/datasets/scannetpp.py

@@ -0,0 +1,591 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+ScanNet++ Benchmark dataset implementation.
+
+ScanNet++ is a high-quality indoor RGB-D dataset with iPhone and DSLR images,
+ground truth camera poses from COLMAP, and high-resolution 3D meshes.
+Reference: https://kaldir.vc.in.tum.de/scannetpp/
+
+Evaluation metrics:
+- 3D reconstruction: Accuracy, Completeness, F-score
+- Camera pose estimation: AUC metrics
+"""
+
+import os
+from typing import Dict as TDict
+
+import cv2
+import imageio
+import numpy as np
+import open3d as o3d
+from addict import Dict
+
+from depth_anything_3.bench.dataset import Dataset, _wait_for_file_ready
+from depth_anything_3.bench.registries import MONO_REGISTRY, MV_REGISTRY
+from depth_anything_3.bench.utils import (
+    create_tsdf_volume,
+    fuse_depth_to_tsdf,
+    nn_correspondance,
+    sample_points_from_mesh,
+)
+from depth_anything_3.utils.constants import (
+    SCANNETPP_DOWN_SAMPLE,
+    SCANNETPP_EVAL_DATA_ROOT,
+    SCANNETPP_EVAL_THRESHOLD,
+    SCANNETPP_INPUT_H,
+    SCANNETPP_INPUT_W,
+    SCANNETPP_MAX_DEPTH,
+    SCANNETPP_SAMPLING_NUMBER,
+    SCANNETPP_SCENES,
+    SCANNETPP_SDF_TRUNC,
+    SCANNETPP_VOXEL_LENGTH,
+)
+from depth_anything_3.utils.pose_align import align_poses_umeyama
+from depth_anything_3.utils.read_write_model import read_model
+
+
+@MV_REGISTRY.register(name="scannetpp")
+@MONO_REGISTRY.register(name="scannetpp")
+class ScanNetPP(Dataset):
+    """
+    ScanNet++ Benchmark dataset wrapper for DepthAnything3 evaluation.
+
+    Supports:
+        - Camera pose estimation evaluation (AUC metrics)
+        - 3D reconstruction evaluation (Accuracy, Completeness, F-score)
+        - TSDF-based point cloud fusion
+
+    Dataset structure:
+        scannetpp/data/
+        ├── {scene_id}/
+        │   ├── merge_dslr_iphone/
+        │   │   ├── colmap/sparse_render_rgb/  # COLMAP reconstruction
+        │   │   ├── images/                     # RGB images
+        │   │   └── render_depth/               # GT depth maps
+        │   └── scans/
+        │       └── mesh_aligned_0.05.ply       # Ground truth mesh
+    """
+
+    data_root = SCANNETPP_EVAL_DATA_ROOT
+    SCENES = SCANNETPP_SCENES
+
+    # Input resolution after undistortion and resize
+    input_h = SCANNETPP_INPUT_H
+    input_w = SCANNETPP_INPUT_W
+
+    # Evaluation hyperparameters from constants
+    max_depth = SCANNETPP_MAX_DEPTH
+    sampling_number = SCANNETPP_SAMPLING_NUMBER
+    voxel_length = SCANNETPP_VOXEL_LENGTH
+    sdf_trunc = SCANNETPP_SDF_TRUNC
+    eval_threshold = SCANNETPP_EVAL_THRESHOLD
+    down_sample = SCANNETPP_DOWN_SAMPLE
+
+    def __init__(self):
+        super().__init__()
+        self._scene_cache = {}
+
+    # ------------------------------
+    # Public API
+    # ------------------------------
+
+    def get_data(self, scene: str) -> Dict:
+        """
+        Collect per-view image paths, intrinsics/extrinsics for a scene.
+
+        Only uses iPhone images (not DSLR).
+
+        Args:
+            scene: Scene identifier (e.g., "09c1414f1b")
+
+        Returns:
+            Dict with:
+                - image_files: List[str] - paths to images
+                - extrinsics: np.ndarray [N, 4, 4] - world-to-camera transforms
+                - intrinsics: np.ndarray [N, 3, 3] - camera intrinsics
+                - aux: Dict with gt_mesh_path, dist, roi, cam_hw, etc.
+        """
+        if scene in self._scene_cache:
+            return self._scene_cache[scene]
+
+        input_path = os.path.join(self.data_root, scene, "merge_dslr_iphone")
+        colmap_path = os.path.join(input_path, "colmap/sparse_render_rgb")
+        image_path = os.path.join(input_path, "images")
+        depth_path_dir = os.path.join(input_path, "render_depth")
+
+        # Read COLMAP model
+        cams, images, points3d = read_model(colmap_path)
+
+        # Map image names to IDs
+        name2id = {image.name: k for k, image in images.items()}
+        names = sorted([image.name for k, image in images.items()])
+        # Only use iPhone images
+        names = [name for name in names if "iphone" in name]
+
+        gt_mesh_path = os.path.join(
+            input_path.replace("merge_dslr_iphone", "scans"), "mesh_aligned_0.05.ply"
+        )
+
+        out = Dict({
+            "image_files": [],
+            "extrinsics": [],
+            "intrinsics": [],
+            "aux": Dict({
+                "gt_mesh_path": gt_mesh_path,
+                "dist_list": [],
+                "roi_list": [],
+                "cam_hw_list": [],
+                "ixt_raw_list": [],
+                "gt_depth_files": [],
+            }),
+        })
+
+        for name in names:
+            image = images[name2id[name]]
+            img_path = os.path.join(image_path, name)
+
+            if not os.path.exists(img_path):
+                continue
+
+            # Build extrinsics (world-to-camera)
+            ext = np.eye(4, dtype=np.float32)
+            ext[:3, :3] = image.qvec2rotmat()
+            ext[:3, 3] = image.tvec
+
+            # Get camera parameters
+            cam_id = image.camera_id
+            camera = cams[cam_id]
+            cam_height, cam_width = camera.height, camera.width
+
+            # Build intrinsics
+            ixt = np.eye(3, dtype=np.float32)
+            ixt[0, 0], ixt[1, 1], ixt[0, 2], ixt[1, 2] = camera.params[:4]
+            ixt[:2, 2] -= 0.5  # COLMAP convention adjustment
+            ixt_raw = ixt.copy()
+
+            # Handle distortion (OPENCV model)
+            dist = np.zeros(5, dtype=np.float32)
+            roi = (0, 0, cam_width, cam_height)
+            if camera.model == "OPENCV":
+                dist[:4] = camera.params[4:]
+                ixt, roi = cv2.getOptimalNewCameraMatrix(
+                    ixt, dist, (cam_width, cam_height), 1, (cam_width, cam_height)
+                )
+
+            # Depth file path
+            frame_name = os.path.basename(name)[:-4]  # Remove .jpg
+            depth_file = os.path.join(depth_path_dir, f"{frame_name}.png")
+
+            out.image_files.append(img_path)
+            out.extrinsics.append(ext)
+            out.intrinsics.append(ixt)
+            out.aux.dist_list.append(dist)
+            out.aux.roi_list.append(roi)
+            out.aux.cam_hw_list.append((cam_height, cam_width))
+            out.aux.ixt_raw_list.append(ixt_raw)
+            out.aux.gt_depth_files.append(depth_file)
+
+        out.extrinsics = np.asarray(out.extrinsics, dtype=np.float32)
+        out.intrinsics = np.asarray(out.intrinsics, dtype=np.float32)
+
+        print(f"[ScanNet++] {scene}: {len(out.image_files)} images")
+
+        self._scene_cache[scene] = out
+        return out
+
+    def load_image(self, img_path: str, idx: int, aux: Dict) -> np.ndarray:
+        """
+        Load and preprocess image with undistortion and cropping.
+
+        Args:
+            img_path: Path to image file
+            idx: Index of the image in the dataset
+            aux: Auxiliary data from get_data
+
+        Returns:
+            Preprocessed RGB image
+        """
+        image = imageio.imread(img_path).astype(np.uint8)
+        ixt_raw = aux.ixt_raw_list[idx]
+        ixt = aux.intrinsics[idx] if hasattr(aux, 'intrinsics') else None
+        dist = aux.dist_list[idx]
+        roi = aux.roi_list[idx]
+
+        # Undistort using raw intrinsics
+        # Use the stored intrinsics from get_data for newCameraMatrix
+        stored_ixt = self._scene_cache.get(aux.scene, {}).get('intrinsics', [None])[idx] if hasattr(aux, 'scene') else None
+        if stored_ixt is None:
+            # Recompute optimal camera matrix for undistortion
+            cam_h, cam_w = aux.cam_hw_list[idx]
+            ixt_for_undistort = ixt_raw.copy()
+            ixt_for_undistort, _ = cv2.getOptimalNewCameraMatrix(
+                ixt_raw, dist, (cam_w, cam_h), 1, (cam_w, cam_h)
+            )
+        else:
+            ixt_for_undistort = stored_ixt
+
+        image = cv2.undistort(image, ixt_raw, dist, newCameraMatrix=ixt_for_undistort)
+
+        # Crop to ROI
+        x, y, w, h = roi
+        image = image[y:y+h, x:x+w]
+
+        # Resize to target resolution
+        image = cv2.resize(image, (self.input_w, self.input_h), interpolation=cv2.INTER_AREA)
+
+        return image
+
+    def eval3d(self, scene: str, fuse_path: str) -> TDict[str, float]:
+        """
+        Evaluate fused point cloud against ScanNet++ ground truth mesh.
+
+        Uses AABB cropping to only evaluate points within GT bounding box.
+
+        Args:
+            scene: Scene identifier
+            fuse_path: Path to fused point cloud (.ply)
+
+        Returns:
+            Dict with metrics: acc, comp, overall, precision, recall, fscore
+        """
+        gt_data = self.get_data(scene)
+        gt_mesh_path = gt_data.aux.gt_mesh_path
+
+        # Load ground truth mesh and sample points
+        gt_mesh = o3d.io.read_triangle_mesh(gt_mesh_path)
+        gt_pcd = sample_points_from_mesh(gt_mesh, self.sampling_number)
+
+        # Load predicted point cloud
+        pred_pcd = o3d.io.read_point_cloud(fuse_path)
+
+        # Crop prediction to GT bounding box (with 0.1m margin)
+        aabb = gt_pcd.get_axis_aligned_bounding_box()
+        points = np.asarray(pred_pcd.points)
+        inside_mask = (
+            (points[:, 0] >= aabb.min_bound[0] - 0.1) &
+            (points[:, 0] <= aabb.max_bound[0] + 0.1) &
+            (points[:, 1] >= aabb.min_bound[1] - 0.1) &
+            (points[:, 1] <= aabb.max_bound[1] + 0.1) &
+            (points[:, 2] >= aabb.min_bound[2] - 0.1) &
+            (points[:, 2] <= aabb.max_bound[2] + 0.1)
+        )
+        pred_pcd = pred_pcd.select_by_index(inside_mask.nonzero()[0])
+
+        # Downsample
+        if self.down_sample > 0:
+            pred_pcd = pred_pcd.voxel_down_sample(self.down_sample)
+            gt_pcd = gt_pcd.voxel_down_sample(self.down_sample)
+
+        verts_pred = np.asarray(pred_pcd.points)
+        verts_gt = np.asarray(gt_pcd.points)
+
+        if len(verts_pred) == 0 or len(verts_gt) == 0:
+            return {
+                "acc": float("inf"),
+                "comp": float("inf"),
+                "overall": float("inf"),
+                "precision": 0.0,
+                "recall": 0.0,
+                "fscore": 0.0,
+            }
+
+        # Compute distances
+        dist_pred_to_gt = nn_correspondance(verts_gt, verts_pred)
+        dist_gt_to_pred = nn_correspondance(verts_pred, verts_gt)
+
+        # Compute metrics
+        accuracy = float(np.mean(dist_pred_to_gt))
+        completeness = float(np.mean(dist_gt_to_pred))
+        overall = (accuracy + completeness) / 2
+
+        precision = float(np.mean((dist_pred_to_gt < self.eval_threshold).astype(float)))
+        recall = float(np.mean((dist_gt_to_pred < self.eval_threshold).astype(float)))
+
+        if precision + recall > 0:
+            fscore = 2 * precision * recall / (precision + recall)
+        else:
+            fscore = 0.0
+
+        return {
+            "acc": accuracy,
+            "comp": completeness,
+            "overall": overall,
+            "precision": precision,
+            "recall": recall,
+            "fscore": fscore,
+        }
+
+    def _load_gt_meta(self, result_path: str) -> Dict:
+        """Load saved GT meta for fusion."""
+        export_dir = os.path.dirname(result_path)
+        gt_meta_path = os.path.join(os.path.dirname(export_dir), "gt_meta.npz")
+
+        if os.path.exists(gt_meta_path):
+            data = np.load(gt_meta_path, allow_pickle=True)
+            image_files = list(data["image_files"])
+
+            # Reconstruct aux data from image files
+            return Dict({
+                "extrinsics": data["extrinsics"],
+                "intrinsics": data["intrinsics"],
+                "image_files": image_files,
+            })
+        return None
+
+    def fuse3d(self, scene: str, result_path: str, fuse_path: str, mode: str) -> None:
+        """
+        Fuse per-view depths into a point cloud using TSDF fusion.
+
+        Args:
+            scene: Scene identifier
+            result_path: Path to npz file with predicted depths/poses
+            fuse_path: Output path for fused point cloud (.ply)
+            mode: "recon_unposed" or "recon_posed"
+        """
+        # Get GT data
+        full_gt_data = self.get_data(scene)
+
+        # Try to load saved GT meta (handles frame sampling)
+        gt_meta = self._load_gt_meta(result_path)
+        if gt_meta is not None:
+            gt_data = gt_meta
+            # Need to rebuild aux from full GT data based on image indices
+            image_indices = [
+                full_gt_data.image_files.index(f)
+                for f in gt_data.image_files
+                if f in full_gt_data.image_files
+            ]
+        else:
+            gt_data = full_gt_data
+            image_indices = list(range(len(full_gt_data.image_files)))
+
+        _wait_for_file_ready(result_path)
+        pred_data = Dict({k: v for k, v in np.load(result_path).items()})
+
+        # Load and preprocess images
+        images = []
+        for idx, img_idx in enumerate(image_indices):
+            img_path = full_gt_data.image_files[img_idx]
+            image = imageio.imread(img_path).astype(np.uint8)
+
+            # Undistort and crop
+            ixt_raw = full_gt_data.aux.ixt_raw_list[img_idx]
+            ixt = full_gt_data.intrinsics[img_idx]
+            dist = full_gt_data.aux.dist_list[img_idx]
+            roi = full_gt_data.aux.roi_list[img_idx]
+
+            image = cv2.undistort(image, ixt_raw, dist, newCameraMatrix=ixt)
+            x, y, w, h = roi
+            image = image[y:y+h, x:x+w]
+            image = cv2.resize(image, (self.input_w, self.input_h), interpolation=cv2.INTER_AREA)
+
+            images.append(image)
+
+        images = np.stack(images, axis=0)
+
+        # Prepare depths, intrinsics, extrinsics
+        if mode == "recon_unposed":
+            depths, intrinsics, extrinsics = self._prep_unposed(
+                pred_data, gt_data, full_gt_data, image_indices, scene=scene
+            )
+        elif mode == "recon_posed":
+            depths, intrinsics, extrinsics = self._prep_posed(
+                pred_data, gt_data, full_gt_data, image_indices, scene=scene
+            )
+        else:
+            raise ValueError(f"Invalid mode: {mode}")
+
+        # Create TSDF volume and fuse
+        volume = create_tsdf_volume(
+            voxel_length=self.voxel_length,
+            sdf_trunc=self.sdf_trunc,
+        )
+        mesh = fuse_depth_to_tsdf(
+            volume, depths, images, intrinsics, extrinsics, max_depth=self.max_depth
+        )
+
+        # Sample points from mesh
+        pcd = sample_points_from_mesh(mesh, self.sampling_number)
+
+        # Save point cloud
+        os.makedirs(os.path.dirname(fuse_path), exist_ok=True)
+        o3d.io.write_point_cloud(fuse_path, pcd)
+
+    # ------------------------------
+    # Private helpers
+    # ------------------------------
+
+    def _prep_unposed(
+        self, pred_data: Dict, gt_data: Dict, full_gt_data: Dict,
+        image_indices: list, scene: str = None
+    ) -> tuple:
+        """Prepare depths/intrinsics/extrinsics for recon_unposed mode."""
+        # Scale alignment with fixed random_state for reproducibility
+        _, _, scale, extrinsics = align_poses_umeyama(
+            gt_data.extrinsics.copy(),
+            pred_data.extrinsics.copy(),
+            return_aligned=True,
+            ransac=True,
+            random_state=42,
+        )
+
+        model_h, model_w = pred_data.depth.shape[1], pred_data.depth.shape[2]
+
+        depths_out = []
+        intrinsics_out = []
+        for i in range(len(pred_data.depth)):
+            img_idx = image_indices[i]
+
+            # Get original image size (after undistort+crop, before resize to input_h/w)
+            orig_h, orig_w = full_gt_data.aux.cam_hw_list[img_idx]
+
+            # Step 1: nearest resize to original image size
+            depth = cv2.resize(
+                pred_data.depth[i],
+                (orig_w, orig_h),
+                interpolation=cv2.INTER_NEAREST,
+            )
+
+            # Step 2: linear resize to target resolution
+            depth = cv2.resize(
+                depth,
+                (self.input_w, self.input_h),
+                interpolation=cv2.INTER_LINEAR,
+            ).astype(np.float32)
+
+            # Load GT depth for masking
+            gt_zero_mask = self._load_gt_mask(full_gt_data.aux.gt_depth_files[img_idx])
+
+            # Mask invalid depths BEFORE scale
+            depth = self._mask_invalid_depth(depth, gt_zero_mask)
+
+            # Apply scale AFTER mask
+            depth = depth * scale
+
+            # Adjust intrinsics to target resolution
+            h_ratio = self.input_h / model_h
+            w_ratio = self.input_w / model_w
+            ixt = pred_data.intrinsics[i].copy()
+            ixt[0, :] *= w_ratio
+            ixt[1, :] *= h_ratio
+
+            depths_out.append(depth)
+            intrinsics_out.append(ixt)
+
+        return np.stack(depths_out), np.stack(intrinsics_out), extrinsics
+
+    def _prep_posed(
+        self, pred_data: Dict, gt_data: Dict, full_gt_data: Dict,
+        image_indices: list, scene: str = None
+    ) -> tuple:
+        """Prepare depths/intrinsics/extrinsics for recon_posed mode."""
+        # Scale alignment
+        _, _, scale, _ = align_poses_umeyama(
+            gt_data.extrinsics.copy(),
+            pred_data.extrinsics.copy(),
+            return_aligned=True,
+            ransac=True,
+            random_state=42,
+        )
+
+        depths_out = []
+        intrinsics_out = []
+        extrinsics_out = []
+
+        for i in range(len(pred_data.depth)):
+            img_idx = image_indices[i]
+
+            # Get original image size (after undistort+crop, before resize to input_h/w)
+            orig_h, orig_w = full_gt_data.aux.cam_hw_list[img_idx]
+
+            # Step 1: nearest resize to original image size
+            depth = cv2.resize(
+                pred_data.depth[i],
+                (orig_w, orig_h),
+                interpolation=cv2.INTER_NEAREST,
+            )
+
+            # Step 2: linear resize to target resolution
+            depth = cv2.resize(
+                depth,
+                (self.input_w, self.input_h),
+                interpolation=cv2.INTER_LINEAR,
+            ).astype(np.float32)
+
+            # Load GT depth for masking
+            gt_zero_mask = self._load_gt_mask(full_gt_data.aux.gt_depth_files[img_idx])
+
+            # Mask invalid depths BEFORE scale
+            depth = self._mask_invalid_depth(depth, gt_zero_mask)
+
+            # Apply scale AFTER mask
+            depth = depth * scale
+
+            depths_out.append(depth)
+
+            # Get GT intrinsics and scale to target resolution
+            ixt = full_gt_data.intrinsics[img_idx].copy()
+            cam_h, cam_w = full_gt_data.aux.cam_hw_list[img_idx]
+            ixt[:2, 2] += 0.5  # Undo COLMAP convention
+            ixt[0, :] *= self.input_w / cam_w
+            ixt[1, :] *= self.input_h / cam_h
+            intrinsics_out.append(ixt)
+
+            extrinsics_out.append(full_gt_data.extrinsics[img_idx])
+
+        return np.stack(depths_out), np.stack(intrinsics_out), np.stack(extrinsics_out)
+
+    def _load_gt_mask(self, gt_depth_path: str) -> np.ndarray:
+        """
+        Load GT depth and create valid mask.
+
+        For ScanNet++, GT depth is stored as 16-bit PNG in millimeters.
+
+        Returns:
+            Boolean mask where True = valid region to keep
+        """
+        if not os.path.exists(gt_depth_path):
+            return None
+
+        gt_depth = imageio.imread(gt_depth_path) / 1000.0  # mm to meters
+
+        # Resize to target resolution
+        gt_depth = cv2.resize(
+            gt_depth,
+            (self.input_w, self.input_h),
+            interpolation=cv2.INTER_LINEAR,
+        ).astype(np.float32)
+
+        # Valid mask: depth > 0 and not inf
+        valid_mask = np.logical_and(gt_depth > 0, gt_depth != np.inf)
+        return valid_mask
+
+    def _mask_invalid_depth(
+        self, depth: np.ndarray, gt_zero_mask: np.ndarray = None
+    ) -> np.ndarray:
+        """Mask invalid depth values by setting them to 0."""
+        depth = depth.copy()
+
+        if gt_zero_mask is not None:
+            pred_invalid = np.isnan(depth) | np.isinf(depth)
+            combined_mask = np.logical_and(gt_zero_mask, np.logical_not(pred_invalid))
+            depth = depth * combined_mask.astype(np.float32)
+        else:
+            invalid_mask = np.isnan(depth) | np.isinf(depth) | (depth <= 0)
+            depth[invalid_mask] = 0.0
+
+        return depth
+

depth_anything_3/bench/datasets/sevenscenes.py

@@ -0,0 +1,449 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+7Scenes Benchmark dataset implementation.
+
+7Scenes is an indoor RGB-D dataset with ground truth camera poses and 3D meshes.
+Reference: https://www.microsoft.com/en-us/research/project/rgb-d-dataset-7-scenes/
+
+Evaluation metrics:
+- 3D reconstruction: Accuracy, Completeness, F-score
+- Camera pose estimation: AUC metrics
+"""
+
+import os
+from typing import Dict as TDict
+
+import cv2
+import numpy as np
+import open3d as o3d
+from addict import Dict
+
+from depth_anything_3.bench.dataset import Dataset, _wait_for_file_ready
+from depth_anything_3.bench.registries import MONO_REGISTRY, MV_REGISTRY
+from depth_anything_3.bench.utils import (
+    create_tsdf_volume,
+    evaluate_3d_reconstruction,
+    fuse_depth_to_tsdf,
+    sample_points_from_mesh,
+)
+from depth_anything_3.utils.constants import (
+    SEVENSCENES_CX,
+    SEVENSCENES_CY,
+    SEVENSCENES_DOWN_SAMPLE,
+    SEVENSCENES_EVAL_DATA_ROOT,
+    SEVENSCENES_EVAL_THRESHOLD,
+    SEVENSCENES_FX,
+    SEVENSCENES_FY,
+    SEVENSCENES_MAX_DEPTH,
+    SEVENSCENES_SAMPLING_NUMBER,
+    SEVENSCENES_SCENES,
+    SEVENSCENES_SDF_TRUNC,
+    SEVENSCENES_VOXEL_LENGTH,
+)
+from depth_anything_3.utils.pose_align import align_poses_umeyama
+
+
+@MV_REGISTRY.register(name="7scenes")
+@MONO_REGISTRY.register(name="7scenes")
+class SevenScenes(Dataset):
+    """
+    7Scenes Benchmark dataset wrapper for DepthAnything3 evaluation.
+
+    Supports:
+        - Camera pose estimation evaluation (AUC metrics)
+        - 3D reconstruction evaluation (Accuracy, Completeness, F-score)
+        - TSDF-based point cloud fusion
+
+    Dataset structure:
+        7scenes/
+        ├── 7Scenes/
+        │   ├── {scene}/
+        │   │   └── seq-01/  (or seq-02 for stairs)
+        │   │       ├── frame-XXXXXX.color.png
+        │   │       ├── frame-XXXXXX.depth.png
+        │   │       └── frame-XXXXXX.pose.txt
+        │   └── meshes/
+        │       └── {scene}.ply  # Ground truth mesh
+    """
+
+    data_root = SEVENSCENES_EVAL_DATA_ROOT
+    SCENES = SEVENSCENES_SCENES
+
+    # Evaluation hyperparameters from constants
+    max_depth = SEVENSCENES_MAX_DEPTH
+    sampling_number = SEVENSCENES_SAMPLING_NUMBER
+    voxel_length = SEVENSCENES_VOXEL_LENGTH
+    sdf_trunc = SEVENSCENES_SDF_TRUNC
+    eval_threshold = SEVENSCENES_EVAL_THRESHOLD
+    down_sample = SEVENSCENES_DOWN_SAMPLE
+
+    # Fixed camera intrinsics for all 7Scenes images
+    fx = SEVENSCENES_FX
+    fy = SEVENSCENES_FY
+    cx = SEVENSCENES_CX
+    cy = SEVENSCENES_CY
+
+    def __init__(self):
+        super().__init__()
+        self._scene_cache = {}
+
+    # ------------------------------
+    # Public API
+    # ------------------------------
+
+    def get_data(self, scene: str) -> Dict:
+        """
+        Collect per-view image paths, intrinsics/extrinsics for a scene.
+
+        Args:
+            scene: Scene identifier (e.g., "chess")
+
+        Returns:
+            Dict with:
+                - image_files: List[str] - paths to images
+                - extrinsics: np.ndarray [N, 4, 4] - world-to-camera transforms
+                - intrinsics: np.ndarray [N, 3, 3] - camera intrinsics
+                - aux: Dict with gt_mesh_path, gt_depth_files
+        """
+        if scene in self._scene_cache:
+            return self._scene_cache[scene]
+
+        # Different sequence for stairs scene
+        if scene == "stairs":
+            data_folder = os.path.join(self.data_root, "7Scenes", scene, "seq-02")
+            n_imgs = 500
+        else:
+            data_folder = os.path.join(self.data_root, "7Scenes", scene, "seq-01")
+            n_imgs = 1000
+
+        gt_mesh_path = os.path.join(self.data_root, "7Scenes", "meshes", f"{scene}.ply")
+
+        # Fixed intrinsics for all images
+        ixt = np.array([
+            [self.fx, 0, self.cx],
+            [0, self.fy, self.cy],
+            [0, 0, 1],
+        ], dtype=np.float32)
+
+        out = Dict({
+            "image_files": [],
+            "extrinsics": [],
+            "intrinsics": [],
+            "aux": Dict({
+                "gt_mesh_path": gt_mesh_path,
+                "gt_depth_files": [],
+            }),
+        })
+
+        for i in range(0, n_imgs, 1):
+            img_path = os.path.join(data_folder, f"frame-{i:06d}.color.png")
+            pose_path = os.path.join(data_folder, f"frame-{i:06d}.pose.txt")
+            depth_path = os.path.join(data_folder, f"frame-{i:06d}.depth.png")
+
+            if not os.path.exists(img_path) or not os.path.exists(pose_path):
+                continue
+
+            # Load camera-to-world pose and convert to world-to-camera (extrinsic)
+            c2w = np.loadtxt(pose_path)
+            ext = np.linalg.inv(c2w).astype(np.float32)
+
+            out.image_files.append(img_path)
+            out.extrinsics.append(ext)
+            out.intrinsics.append(ixt.copy())
+            out.aux.gt_depth_files.append(depth_path)
+
+        out.extrinsics = np.asarray(out.extrinsics, dtype=np.float32)
+        out.intrinsics = np.asarray(out.intrinsics, dtype=np.float32)
+
+        print(f"[7Scenes] {scene}: {len(out.image_files)} images")
+
+        self._scene_cache[scene] = out
+        return out
+
+    def eval3d(self, scene: str, fuse_path: str) -> TDict[str, float]:
+        """
+        Evaluate fused point cloud against 7Scenes ground truth mesh.
+
+        Args:
+            scene: Scene identifier
+            fuse_path: Path to fused point cloud (.ply)
+
+        Returns:
+            Dict with metrics: acc, comp, overall, precision, recall, fscore
+        """
+        gt_data = self.get_data(scene)
+        gt_mesh_path = gt_data.aux.gt_mesh_path
+
+        # Load and sample ground truth mesh
+        gt_mesh = o3d.io.read_triangle_mesh(gt_mesh_path)
+        gt_pcd = sample_points_from_mesh(gt_mesh, self.sampling_number)
+
+        # Load predicted point cloud
+        pred_pcd = o3d.io.read_point_cloud(fuse_path)
+
+        # Evaluate using shared utility function
+        metrics = evaluate_3d_reconstruction(
+            pred_pcd,
+            gt_pcd,
+            threshold=self.eval_threshold,
+            down_sample=self.down_sample,
+        )
+
+        return metrics
+
+    def _load_gt_meta(self, result_path: str) -> Dict:
+        """
+        Load saved GT meta (extrinsics, intrinsics, image_files) for fusion.
+
+        This is needed when frames are sampled, so fuse3d uses the correct
+        (sampled) GT instead of full dataset GT.
+
+        Args:
+            result_path: Path to npz file (used to derive gt_meta.npz path)
+
+        Returns:
+            Dict with GT data, or None if gt_meta.npz doesn't exist
+        """
+        export_dir = os.path.dirname(result_path)  # exports/mini_npz/
+        gt_meta_path = os.path.join(os.path.dirname(export_dir), "gt_meta.npz")
+
+        if os.path.exists(gt_meta_path):
+            data = np.load(gt_meta_path, allow_pickle=True)
+            # Build aux with gt_depth_files derived from image_files
+            image_files = list(data["image_files"])
+            gt_depth_files = [
+                img_path.replace("color", "depth").replace(".color.", ".depth.")
+                for img_path in image_files
+            ]
+            return Dict({
+                "extrinsics": data["extrinsics"],
+                "intrinsics": data["intrinsics"],
+                "image_files": image_files,
+                "aux": Dict({"gt_depth_files": gt_depth_files}),
+            })
+        return None
+
+    def fuse3d(self, scene: str, result_path: str, fuse_path: str, mode: str) -> None:
+        """
+        Fuse per-view depths into a point cloud using TSDF fusion.
+
+        Args:
+            scene: Scene identifier
+            result_path: Path to npz file with predicted depths/poses
+            fuse_path: Output path for fused point cloud (.ply)
+            mode: "recon_unposed" or "recon_posed"
+        """
+        # Try to load saved GT meta (handles frame sampling)
+        gt_meta = self._load_gt_meta(result_path)
+        if gt_meta is not None:
+            gt_data = gt_meta
+        else:
+            gt_data = self.get_data(scene)
+        _wait_for_file_ready(result_path)
+        pred_data = Dict({k: v for k, v in np.load(result_path).items()})
+
+        # Load original images (keep original size)
+        images = []
+        orig_sizes = []
+        for img_path in gt_data.image_files:
+            img = cv2.imread(img_path)
+            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+            images.append(img)
+            orig_sizes.append((img.shape[0], img.shape[1]))
+
+        # Prepare depths, intrinsics, extrinsics
+        if mode == "recon_unposed":
+            depths, intrinsics, extrinsics = self._prep_unposed(
+                pred_data, gt_data, orig_sizes, scene=scene
+            )
+        elif mode == "recon_posed":
+            depths, intrinsics, extrinsics = self._prep_posed(
+                pred_data, gt_data, orig_sizes, scene=scene
+            )
+        else:
+            raise ValueError(f"Invalid mode: {mode}")
+
+        images = np.stack(images, axis=0)
+
+        # Create TSDF volume and fuse
+        volume = create_tsdf_volume(
+            voxel_length=self.voxel_length,
+            sdf_trunc=self.sdf_trunc,
+        )
+        mesh = fuse_depth_to_tsdf(
+            volume, depths, images, intrinsics, extrinsics, max_depth=self.max_depth
+        )
+
+        # Sample points from mesh
+        pcd = sample_points_from_mesh(mesh, self.sampling_number)
+
+        # Save point cloud
+        os.makedirs(os.path.dirname(fuse_path), exist_ok=True)
+        o3d.io.write_point_cloud(fuse_path, pcd)
+
+    # ------------------------------
+    # Private helpers
+    # ------------------------------
+
+    def _prep_unposed(
+        self, pred_data: Dict, gt_data: Dict, orig_sizes: list, scene: str
+    ) -> tuple:
+        """
+        Prepare depths/intrinsics/extrinsics for recon_unposed mode.
+
+        Similar to ETH3D but uses GT depth for masking instead of separate mask files.
+        """
+        # Scale alignment with fixed random_state for reproducibility
+        _, _, scale, extrinsics = align_poses_umeyama(
+            gt_data.extrinsics.copy(),
+            pred_data.extrinsics.copy(),
+            return_aligned=True,
+            ransac=True,
+            random_state=42,
+        )
+
+        model_h, model_w = pred_data.depth.shape[1], pred_data.depth.shape[2]
+
+        depths_out = []
+        intrinsics_out = []
+        for i in range(len(pred_data.depth)):
+            orig_h, orig_w = orig_sizes[i]
+
+            # Resize depth to original image size (nearest interpolation)
+            depth = cv2.resize(
+                pred_data.depth[i],
+                (orig_w, orig_h),
+                interpolation=cv2.INTER_NEAREST,
+            )
+
+            # Load GT depth for masking
+            gt_zero_mask = self._load_gt_mask(gt_data.aux.gt_depth_files[i])
+
+            # Mask invalid depths BEFORE scale
+            depth = self._mask_invalid_depth(depth, gt_zero_mask)
+
+            # Apply scale AFTER mask
+            depth = depth * scale
+
+            # Adjust intrinsics to original image size
+            h_ratio = orig_h / model_h
+            w_ratio = orig_w / model_w
+            ixt = pred_data.intrinsics[i].copy()
+            ixt[0, :] *= w_ratio
+            ixt[1, :] *= h_ratio
+
+            depths_out.append(depth)
+            intrinsics_out.append(ixt)
+
+        return np.stack(depths_out), np.stack(intrinsics_out), extrinsics
+
+    def _prep_posed(
+        self, pred_data: Dict, gt_data: Dict, orig_sizes: list, scene: str
+    ) -> tuple:
+        """
+        Prepare depths/intrinsics/extrinsics for recon_posed mode.
+        Uses GT intrinsics/extrinsics but aligns depth scale via Umeyama.
+        """
+        # Scale alignment with fixed random_state
+        _, _, scale, _ = align_poses_umeyama(
+            gt_data.extrinsics.copy(),
+            pred_data.extrinsics.copy(),
+            return_aligned=True,
+            ransac=True,
+            random_state=42,
+        )
+
+        model_h, model_w = pred_data.depth.shape[1], pred_data.depth.shape[2]
+
+        depths_out = []
+        for i in range(len(pred_data.depth)):
+            orig_h, orig_w = orig_sizes[i]
+
+            # Resize depth to original image size
+            depth = cv2.resize(
+                pred_data.depth[i],
+                (orig_w, orig_h),
+                interpolation=cv2.INTER_NEAREST,
+            )
+
+            # Load GT depth for masking
+            gt_zero_mask = self._load_gt_mask(gt_data.aux.gt_depth_files[i])
+
+            # Mask invalid depths BEFORE scale
+            depth = self._mask_invalid_depth(depth, gt_zero_mask)
+
+            # Apply scale AFTER mask
+            depth = depth * scale
+
+            depths_out.append(depth)
+
+        # Use GT intrinsics and extrinsics
+        return np.stack(depths_out), gt_data.intrinsics.copy(), gt_data.extrinsics.copy()
+
+    def _load_gt_mask(self, gt_depth_path: str) -> np.ndarray:
+        """
+        Load GT depth and create valid mask.
+
+        For 7Scenes, GT depth is stored as 16-bit PNG in millimeters.
+        Value 65535 indicates invalid depth.
+
+        Returns:
+            Boolean mask where True = valid region to keep
+        """
+        if not os.path.exists(gt_depth_path):
+            return None
+
+        gt_depth = cv2.imread(gt_depth_path, -1)
+        if gt_depth is None:
+            return None
+
+        # 65535 is invalid depth marker in 7Scenes
+        gt_depth[gt_depth == 65535] = 0
+        # Convert to meters
+        gt_depth = gt_depth / 1000.0
+
+        # Valid mask: depth > 0
+        valid_mask = gt_depth > 0
+        return valid_mask
+
+    def _mask_invalid_depth(
+        self, depth: np.ndarray, gt_zero_mask: np.ndarray = None
+    ) -> np.ndarray:
+        """
+        Mask invalid depth values by setting them to 0.
+
+        Args:
+            depth: Depth map to mask
+            gt_zero_mask: Optional GT mask (True = valid region)
+
+        Returns:
+            Masked depth map with invalid regions set to 0
+        """
+        depth = depth.copy()
+
+        if gt_zero_mask is not None:
+            # Also mask out invalid pred depth
+            pred_invalid = np.isnan(depth) | np.isinf(depth)
+            combined_mask = np.logical_and(gt_zero_mask, np.logical_not(pred_invalid))
+            depth = depth * combined_mask.astype(np.float32)
+        else:
+            # Fallback: only mask pred invalid values
+            invalid_mask = np.isnan(depth) | np.isinf(depth) | (depth <= 0)
+            depth[invalid_mask] = 0.0
+
+        return depth
+
+

depth_anything_3/bench/evaluator.py

@@ -0,0 +1,752 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Main Evaluator class for DepthAnything3 benchmark evaluation.
+
+Supports multiple datasets and evaluation modes:
+- pose: Camera pose estimation (AUC metrics)
+- recon_unposed: 3D reconstruction with predicted poses
+- recon_posed: 3D reconstruction with GT poses
+- view_syn: Novel view synthesis (TODO)
+"""
+
+import json
+import os
+import random
+from typing import Dict as TDict, Iterable, List
+
+import numpy as np
+import torch
+from addict import Dict
+from tqdm import tqdm
+
+from depth_anything_3.bench.print_metrics import MetricsPrinter
+from depth_anything_3.utils.parallel_utils import parallel_execution
+from depth_anything_3.bench.registries import MV_REGISTRY
+from depth_anything_3.utils.constants import EVAL_REF_VIEW_STRATEGY
+
+
+class Evaluator:
+    """
+    Main evaluation orchestrator for DepthAnything3 benchmarks.
+
+    Usage:
+        evaluator = Evaluator(
+            work_dir="./eval_workspace",
+            datas=["dtu"],
+            modes=["pose", "recon_unposed", "recon_posed"],
+        )
+        api = DepthAnything3.from_pretrained("...")
+        evaluator.infer(api)
+        metrics = evaluator.eval()
+        evaluator.print_metrics()
+    """
+
+    VALID_MODES = {"pose", "recon_unposed", "recon_posed", "view_syn"}
+
+    def __init__(
+        self,
+        work_dir: str = "./eval_workspace",
+        datas: List[str] = ("dtu",),
+        modes: List[str] = ("recon_unposed",),
+        ref_view_strategy: str = EVAL_REF_VIEW_STRATEGY,
+        scenes: List[str] = None,
+        debug: bool = False,
+        num_fusion_workers: int = 4,
+        max_frames: int = 100,
+        gpu_id: int = 0,
+        total_gpus: int = 1,
+    ):
+        """
+        Initialize the evaluator.
+
+        Args:
+            work_dir: Base directory for model outputs and metric files
+            datas: List of dataset names (must be registered in MV_REGISTRY)
+            modes: List of evaluation modes to run
+            ref_view_strategy: Reference view selection strategy for inference
+                               ("first", "saddle_balanced", etc.)
+            scenes: Specific scenes to evaluate (None = all scenes)
+            debug: Enable verbose debug output
+            num_fusion_workers: Number of parallel workers for TSDF fusion (default: 4)
+            max_frames: Maximum number of frames per scene (default: 100).
+                        If a scene has more frames, randomly sample to this limit.
+                        Set to -1 to disable sampling.
+            gpu_id: GPU index for multi-GPU (0-indexed)
+            total_gpus: Total number of GPUs for task distribution
+        """
+        self.work_dir = work_dir
+        self.datas = list(datas)
+        self.modes = set(modes)
+        self.ref_view_strategy = ref_view_strategy
+        self.scenes_filter = scenes
+        self.debug = debug
+        self.num_fusion_workers = num_fusion_workers
+        self.max_frames = max_frames
+        self.gpu_id = gpu_id
+        self.total_gpus = total_gpus
+
+        # Validate modes
+        unknown = self.modes - self.VALID_MODES
+        if unknown:
+            raise ValueError(f"Unknown modes: {unknown}. Valid: {sorted(self.VALID_MODES)}")
+
+        os.makedirs(self.work_dir, exist_ok=True)
+
+        # Initialize datasets
+        self.datasets = Dict()
+        for data in self.datas:
+            if not MV_REGISTRY.has(data):
+                available = list(MV_REGISTRY.all().keys())
+                raise ValueError(f"Dataset '{data}' not found. Available: {available}")
+            self.datasets[data] = MV_REGISTRY.get(data)()
+
+        # Initialize metrics printer
+        self._printer = MetricsPrinter()
+
+    # -------------------- Public APIs -------------------- #
+
+    def all(self, api) -> TDict[str, dict]:
+        """
+        Run complete evaluation pipeline: inference + evaluation.
+
+        Args:
+            api: DepthAnything3 API instance
+
+        Returns:
+            Combined metrics dictionary
+        """
+        self.infer(api)
+        return self.eval()
+
+    def _get_scenes(self, dataset) -> List[str]:
+        """Get list of scenes to evaluate, optionally filtered."""
+        all_scenes = dataset.SCENES
+        if self.scenes_filter:
+            scenes = [s for s in all_scenes if s in self.scenes_filter]
+            if self.debug:
+                print(f"[DEBUG] Filtered scenes: {scenes} (from {len(all_scenes)} total)")
+            return scenes
+        return all_scenes
+
+    def infer(self, api, model_path: str = None) -> None:
+        """
+        Run inference according to requested modes.
+
+        - Unposed export if 'pose' or 'recon_unposed' is in modes
+        - Posed export if 'recon_posed' or 'view_syn' is in modes
+
+        Multi-GPU: Use --gpu_id and --total_gpus to distribute tasks.
+        Example: Launch 4 processes with gpu_id=0,1,2,3 and total_gpus=4
+
+        Args:
+            api: DepthAnything3 API instance
+            model_path: Model path (unused, kept for API compatibility)
+        """
+        need_unposed = {"pose", "recon_unposed"} & self.modes
+        need_posed = {"recon_posed", "view_syn"} & self.modes
+        export_format = "mini_npz-glb" if self.debug else "mini_npz"
+
+        # Collect all tasks
+        all_tasks = []
+        for data in self.datas:
+            dataset = self.datasets[data]
+            for scene in self._get_scenes(dataset):
+                all_tasks.append((data, scene))
+
+        # Distribute tasks across GPUs
+        if self.total_gpus > 1:
+            tasks = [t for i, t in enumerate(all_tasks) if i % self.total_gpus == self.gpu_id]
+            print(f"[INFO] GPU {self.gpu_id}/{self.total_gpus}: {len(tasks)}/{len(all_tasks)} tasks")
+        else:
+            tasks = all_tasks
+            print(f"[INFO] Total inference tasks: {len(tasks)}")
+
+        for data, scene in tqdm(tasks, desc=f"Inference (GPU {self.gpu_id})"):
+            dataset = self.datasets[data]
+            scene_data = dataset.get_data(scene)
+            scene_data = self._sample_frames(scene_data, scene)
+
+            if need_unposed:
+                export_dir = self._export_dir(data, scene, posed=False)
+                api.inference(
+                    scene_data.image_files,
+                    export_dir=export_dir,
+                    export_format=export_format,
+                    ref_view_strategy=self.ref_view_strategy,
+                )
+                self._save_gt_meta(export_dir, scene_data)
+
+            if need_posed:
+                export_dir = self._export_dir(data, scene, posed=True)
+                api.inference(
+                    scene_data.image_files,
+                    scene_data.extrinsics,
+                    scene_data.intrinsics,
+                    export_dir=export_dir,
+                    export_format=export_format,
+                    ref_view_strategy=self.ref_view_strategy,
+                )
+                self._save_gt_meta(export_dir, scene_data)
+
+    def eval(self) -> TDict[str, dict]:
+        """
+        Evaluate for all configured modes and write JSON files.
+        
+        Evaluation order by mode (all datasets per mode):
+        1. pose - all datasets
+        2. recon_unposed - all datasets
+        3. recon_posed - all datasets
+
+        Returns:
+            Summary mapping: {"<data>_<mode>": metrics_dict}
+        """
+        summary: TDict[str, dict] = {}
+
+        # Evaluate by mode (all datasets per mode)
+        if "pose" in self.modes:
+            print(f"\n{'='*60}")
+            print(f"📊 Evaluating POSE for all datasets...")
+            print(f"{'='*60}")
+            for data, result in self._eval_pose():
+                summary[f"{data}_pose"] = result
+
+        if "recon_unposed" in self.modes:
+            print(f"\n{'='*60}")
+            print(f"📊 Evaluating RECON_UNPOSED for all datasets...")
+            print(f"{'='*60}")
+            for data, result in self._eval_reconstruction("recon_unposed"):
+                summary[f"{data}_recon_unposed"] = result
+
+        if "recon_posed" in self.modes:
+            print(f"\n{'='*60}")
+            print(f"📊 Evaluating RECON_POSED for all datasets...")
+            print(f"{'='*60}")
+            for data, result in self._eval_reconstruction("recon_posed"):
+                summary[f"{data}_recon_posed"] = result
+
+        if "view_syn" in self.modes:
+            # TODO: Add view synthesis metrics here when available
+            pass
+
+        return summary
+
+    def print_metrics(self, metrics: TDict[str, dict] = None) -> None:
+        """
+        Print evaluation metrics in a beautiful tabular format.
+
+        Args:
+            metrics: Metrics dictionary. If None, loads from saved JSON files.
+        """
+        if metrics is None:
+            metrics = self._load_metrics()
+
+        self._printer.print_results(metrics)
+
+    # -------------------- Evaluation Methods -------------------- #
+
+    def _eval_pose(self) -> Iterable[tuple]:
+        """Compute pose-estimation metrics for each dataset and scene."""
+        os.makedirs(self._metric_dir, exist_ok=True)
+
+        for data in tqdm(self.datas, desc="Datasets (pose eval)"):
+            dataset = self.datasets[data]
+            dataset_results = Dict()
+            scenes = self._get_scenes(dataset)
+
+            for scene in tqdm(scenes, desc=f"{data} scenes", leave=False):
+                export_dir = self._export_dir(data, scene, posed=False)
+                result_path = os.path.join(export_dir, "exports", "mini_npz", "results.npz")
+                
+                # Check if result file exists and is valid
+                if not os.path.exists(result_path):
+                    print(f"\n[ERROR] Result file not found: {result_path}")
+                    print(f"[ERROR] CWD: {os.getcwd()}")
+                    print(f"[ERROR] Please run inference first (remove --eval_only)")
+                    continue
+                
+                try:
+                    # Use saved GT meta (handles frame sampling correctly)
+                    gt_meta = self._load_gt_meta(export_dir)
+                    if gt_meta is not None:
+                        result = self._compute_pose_with_gt(result_path, gt_meta)
+                    else:
+                        # Fallback to dataset GT (no sampling was done)
+                        result = dataset.eval_pose(scene, result_path)
+                    dataset_results[scene] = self._to_float_dict(result)
+                except Exception as e:
+                    print(f"\n[ERROR] Failed to evaluate pose for {data}/{scene}: {e}")
+                    print(f"[ERROR] File path: {os.path.abspath(result_path)}")
+                    if self.debug:
+                        import traceback
+                        traceback.print_exc()
+                    continue
+
+            if not dataset_results:
+                print(f"[WARNING] No valid results for {data}")
+                continue
+                
+            dataset_results["mean"] = self._mean_of_dicts(dataset_results.values())
+            out_path = os.path.join(self._metric_dir, f"{data}_pose.json")
+            self._dump_json(out_path, dataset_results)
+            yield data, dataset_results
+
+    def _eval_reconstruction(self, mode: str) -> Iterable[tuple]:
+        """
+        Compute reconstruction metrics for each dataset and scene.
+
+        Args:
+            mode: "recon_unposed" or "recon_posed"
+        """
+        assert mode in {"recon_unposed", "recon_posed"}
+        os.makedirs(self._metric_dir, exist_ok=True)
+
+        posed_flag = mode == "recon_posed"
+        
+        # Filter out datasets that don't support reconstruction (e.g., dtu64)
+        recon_datas = [d for d in self.datas if d != "dtu64"]
+
+        for data in tqdm(recon_datas, desc=f"Datasets ({mode} eval)"):
+            dataset = self.datasets[data]
+            dataset_results = Dict()
+            scenes = self._get_scenes(dataset)
+
+            # Prepare paths for all scenes
+            scene_list = []
+            result_paths = []
+            fuse_paths = []
+            for scene in scenes:
+                export_dir = self._export_dir(data, scene, posed=posed_flag)
+                result_path = os.path.join(export_dir, "exports", "mini_npz", "results.npz")
+                fuse_path = os.path.join(export_dir, "exports", "fuse", "pcd.ply")
+                scene_list.append(scene)
+                result_paths.append(result_path)
+                fuse_paths.append(fuse_path)
+
+            # Parallel fusion (default 4 workers)
+            # DTU uses CUDA operations in fusion, which doesn't work well with ThreadPool
+            use_sequential = (data == "dtu")
+            parallel_execution(
+                scene_list,
+                result_paths,
+                fuse_paths,
+                action=lambda s, rp, fp: dataset.fuse3d(s, rp, fp, mode),
+                num_processes=self.num_fusion_workers,
+                print_progress=True,
+                desc=f"{data} fusion",
+                sequential=use_sequential,
+            )
+
+            # Sequential evaluation (fast, no need to parallelize)
+            for scene, fuse_path in zip(scene_list, fuse_paths):
+                # DTU supports CPU-based evaluation
+                if data == "dtu" and hasattr(dataset, "eval3d"):
+                    result = dataset.eval3d(scene, fuse_path)
+                else:
+                    result = dataset.eval3d(scene, fuse_path)
+                dataset_results[scene] = self._to_float_dict(result)
+                print(f"  {mode} | {data} | {scene}: {result}")
+
+            dataset_results["mean"] = self._mean_of_dicts(dataset_results.values())
+            out_path = os.path.join(self._metric_dir, f"{data}_{mode}.json")
+            self._dump_json(out_path, dataset_results)
+            yield data, dataset_results
+
+    # -------------------- Helpers -------------------- #
+
+    def _save_gt_meta(self, export_dir: str, scene_data: Dict) -> None:
+        """
+        Save GT extrinsics/intrinsics/image_files for evaluation.
+
+        This is needed when frames are sampled, so eval_pose and fuse3d can use
+        the correct (sampled) GT instead of full dataset GT.
+
+        Args:
+            export_dir: Export directory for the scene
+            scene_data: Sampled scene data
+        """
+        meta_path = os.path.join(export_dir, "exports", "gt_meta.npz")
+        os.makedirs(os.path.dirname(meta_path), exist_ok=True)
+        np.savez_compressed(
+            meta_path,
+            extrinsics=scene_data.extrinsics,
+            intrinsics=scene_data.intrinsics,
+            image_files=np.array(scene_data.image_files, dtype=object),
+        )
+
+    def _load_gt_meta(self, export_dir: str) -> Dict:
+        """
+        Load saved GT extrinsics/intrinsics for evaluation.
+
+        Returns:
+            Dict with extrinsics and intrinsics, or None if not found
+        """
+        meta_path = os.path.join(export_dir, "exports", "gt_meta.npz")
+        if os.path.exists(meta_path):
+            data = np.load(meta_path)
+            return Dict({
+                "extrinsics": data["extrinsics"],
+                "intrinsics": data["intrinsics"],
+            })
+        return None
+
+    def _compute_pose_with_gt(self, result_path: str, gt_meta: Dict) -> TDict[str, float]:
+        """
+        Compute pose metrics using saved GT meta (handles frame sampling).
+
+        Args:
+            result_path: Path to npz with predicted extrinsics
+            gt_meta: Dict with GT extrinsics from saved meta
+
+        Returns:
+            Dict with pose metrics
+        """
+        from depth_anything_3.bench.dataset import _wait_for_file_ready
+        from depth_anything_3.bench.utils import compute_pose
+        from depth_anything_3.utils.geometry import as_homogeneous
+
+        _wait_for_file_ready(result_path)
+        pred = np.load(result_path)
+        return compute_pose(
+            torch.from_numpy(as_homogeneous(pred["extrinsics"])),
+            torch.from_numpy(as_homogeneous(gt_meta["extrinsics"])),
+        )
+
+    def _sample_frames(self, scene_data: Dict, scene: str) -> Dict:
+        """
+        Sample frames if scene has more than max_frames.
+
+        Uses fixed random seed (42) for reproducibility.
+
+        Args:
+            scene_data: Scene data dict with image_files, extrinsics, intrinsics, aux
+            scene: Scene name (for logging)
+
+        Returns:
+            Sampled scene_data if num_frames > max_frames, otherwise original
+        """
+        if self.max_frames <= 0:
+            return scene_data
+
+        num_frames = len(scene_data.image_files)
+        if num_frames <= self.max_frames:
+            return scene_data
+
+        # Sample with fixed seed for reproducibility
+        random.seed(42)
+        indices = list(range(num_frames))
+        random.shuffle(indices)
+        sampled_indices = sorted(indices[:self.max_frames])
+
+        print(f"  [Sampling] {scene}: {num_frames} -> {self.max_frames} frames")
+
+        # Create new scene_data with sampled frames
+        sampled = Dict()
+        sampled.image_files = [scene_data.image_files[i] for i in sampled_indices]
+        sampled.extrinsics = scene_data.extrinsics[sampled_indices]
+        sampled.intrinsics = scene_data.intrinsics[sampled_indices]
+
+        # Copy aux data, sampling lists if needed
+        sampled.aux = Dict()
+        for key, val in scene_data.aux.items():
+            if isinstance(val, list) and len(val) == num_frames:
+                sampled.aux[key] = [val[i] for i in sampled_indices]
+            elif isinstance(val, np.ndarray) and len(val) == num_frames:
+                sampled.aux[key] = val[sampled_indices]
+            else:
+                sampled.aux[key] = val
+
+        return sampled
+
+    @property
+    def _metric_dir(self) -> str:
+        """Directory for storing metric JSON files."""
+        return os.path.join(self.work_dir, "metric_results")
+
+    def _export_dir(self, data: str, scene: str, posed: bool) -> str:
+        """
+        Get export directory path.
+
+        Structure: .../model_results/{data}/{scene}/{posed|unposed}
+        """
+        suffix = "posed" if posed else "unposed"
+        export_dir = os.path.join(self.work_dir, "model_results", data, scene, suffix)
+        os.makedirs(export_dir, exist_ok=True)
+        return export_dir
+
+    @staticmethod
+    def _to_float_dict(d: TDict[str, float]) -> dict:
+        """Convert numpy scalars to plain Python floats for JSON safety."""
+        return {k: float(v) for k, v in d.items()}
+
+    @staticmethod
+    def _mean_of_dicts(dicts: Iterable[dict]) -> dict:
+        """Compute elementwise mean across a list of homogeneous metric dicts."""
+        dicts = list(dicts)
+        if not dicts:
+            return {}
+        keys = dicts[0].keys()
+        return {k: float(np.mean([d[k] for d in dicts]).item()) for k in keys}
+
+    @staticmethod
+    def _dump_json(path: str, obj: dict, indent: int = 4) -> None:
+        """Write JSON with UTF-8 and pretty indentation."""
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(obj, f, indent=indent, ensure_ascii=False)
+
+    def _load_metrics(self) -> TDict[str, dict]:
+        """Load evaluation metrics from JSON files."""
+        metrics = {}
+        metric_dir = self._metric_dir
+
+        if not os.path.exists(metric_dir):
+            return metrics
+
+        for filename in os.listdir(metric_dir):
+            if filename.endswith(".json"):
+                filepath = os.path.join(metric_dir, filename)
+                try:
+                    with open(filepath, encoding="utf-8") as f:
+                        data = json.load(f)
+                    key = filename[:-5]  # Remove .json extension
+                    metrics[key] = data
+                except Exception as e:
+                    print(f"Warning: Failed to read metrics file: {filename} - {e}")
+
+        return metrics
+
+
+# -------------------- CLI Entry Point -------------------- #
+
+
+if __name__ == "__main__":
+    import sys
+    from omegaconf import OmegaConf
+    from depth_anything_3.cfg import load_config
+
+    # Get default config path (relative to this file)
+    _default_config = os.path.join(
+        os.path.dirname(__file__), "configs", "eval_bench.yaml"
+    )
+
+    # Check for help flag first (we need to handle this before OmegaConf)
+    if "--help" in sys.argv or "-h" in sys.argv:
+        pass  # Will handle after config loading
+
+    # Set up argv for OmegaConf processing
+    argv = sys.argv[1:]
+
+    # Check if user provides custom config
+    config_path = _default_config
+    if "--config" in argv:
+        config_idx = argv.index("--config")
+        if config_idx + 1 < len(argv):
+            config_path = argv[config_idx + 1]
+            # Remove --config and its value
+            argv = argv[:config_idx] + argv[config_idx + 2:]
+
+    # Print help if requested
+    if "--help" in sys.argv or "-h" in sys.argv:
+        print("""
+DepthAnything3 Benchmark Evaluation
+
+Usage:
+  python -m depth_anything_3.bench.evaluator [OPTIONS] [KEY=VALUE ...]
+
+Configuration:
+  --config PATH                      Config YAML file (default: bench/configs/eval_bench.yaml)
+
+Config Overrides (using dotlist notation):
+  model.path=VALUE                   Model path or HuggingFace ID
+  workspace.work_dir=VALUE           Working directory for outputs
+  eval.datasets=[dataset1,dataset2]  Datasets to evaluate (eth3d,7scenes,scannetpp,hiroom,dtu,dtu64)
+  eval.modes=[mode1,mode2]           Evaluation modes (pose,recon_unposed,recon_posed)
+  eval.scenes=[scene1,scene2]        Specific scenes to evaluate (null=all)
+  eval.max_frames=VALUE              Max frames per scene (-1=no limit, default: 100)
+  eval.ref_view_strategy=VALUE       Reference view strategy (default: first)
+  eval.eval_only=VALUE               Only run evaluation (skip inference) (true/false)
+  eval.print_only=VALUE              Only print saved metrics (true/false)
+  inference.num_fusion_workers=VALUE Number of parallel workers (default: 4)
+  inference.debug=VALUE              Enable debug mode (true/false)
+
+Special Flags:
+  --help, -h                         Show this help message
+
+Multi-GPU:
+  Use CUDA_VISIBLE_DEVICES to specify GPUs (auto-detected and distributed)
+
+Examples:
+  # Use default config
+  python -m depth_anything_3.bench.evaluator
+
+  # Override model path
+  python -m depth_anything_3.bench.evaluator model.path=depth-anything/DA3-LARGE
+
+  # Evaluate specific datasets and modes
+  python -m depth_anything_3.bench.evaluator \\
+      eval.datasets=[eth3d,hiroom] \\
+      eval.modes=[pose]
+
+  # Use custom config with overrides
+  python -m depth_anything_3.bench.evaluator \\
+      --config my_config.yaml \\
+      model.path=/path/to/model \\
+      eval.max_frames=50
+
+  # Multi-GPU inference (auto-distributed)
+  CUDA_VISIBLE_DEVICES=0,1,2,3 python -m depth_anything_3.bench.evaluator
+
+  # Debug specific scenes
+  python -m depth_anything_3.bench.evaluator \\
+      eval.datasets=[eth3d] \\
+      eval.scenes=[courtyard] \\
+      inference.debug=true
+
+  # Only evaluate (skip inference)
+  python -m depth_anything_3.bench.evaluator eval.eval_only=true
+
+  # Only print saved metrics
+  python -m depth_anything_3.bench.evaluator eval.print_only=true
+
+          """)
+        sys.exit(0)
+
+    # Load config with CLI overrides using OmegaConf dotlist
+    # Example: python evaluator.py model.path=/path/to/model eval.datasets=[eth3d,dtu]
+    config = load_config(config_path, argv=argv)
+
+    # Extract config values
+    work_dir = config.workspace.work_dir
+    model_path = config.model.path
+    datasets = config.eval.datasets
+    modes = config.eval.modes
+    ref_view_strategy = config.eval.ref_view_strategy
+    scenes = config.eval.scenes
+    max_frames = config.eval.max_frames
+    eval_only = config.eval.eval_only
+    print_only = config.eval.print_only
+    debug = config.inference.debug
+    num_fusion_workers = config.inference.num_fusion_workers
+
+    # GPU settings: parse from CLI dotlist args (gpu_id=X total_gpus=Y)
+    # These are passed by the main process when spawning workers
+    gpu_id = 0
+    total_gpus = 1
+    for arg in argv:
+        if arg.startswith("gpu_id="):
+            gpu_id = int(arg.split("=")[1])
+        elif arg.startswith("total_gpus="):
+            total_gpus = int(arg.split("=")[1])
+
+    # Override dataset scenes if specified
+    if scenes:
+        print(f"[INFO] Running on specific scenes: {scenes}")
+
+    evaluator = Evaluator(
+        work_dir=work_dir,
+        datas=datasets,
+        modes=modes,
+        ref_view_strategy=ref_view_strategy,
+        scenes=scenes,
+        debug=debug,
+        num_fusion_workers=num_fusion_workers,
+        max_frames=max_frames,
+        gpu_id=gpu_id,
+        total_gpus=total_gpus,
+    )
+
+    if print_only:
+        evaluator.print_metrics()
+    elif eval_only:
+        metrics = evaluator.eval()
+        evaluator.print_metrics(metrics)
+    else:
+        # Parse CUDA_VISIBLE_DEVICES to get GPU list
+        # If not set, use all available GPUs
+        cuda_devices = os.environ.get("CUDA_VISIBLE_DEVICES")
+        if cuda_devices is not None and cuda_devices.strip():
+            gpu_list = [g.strip() for g in cuda_devices.split(",") if g.strip()]
+        else:
+            # CUDA_VISIBLE_DEVICES not set, use all available GPUs
+            num_available = torch.cuda.device_count()
+            gpu_list = [str(i) for i in range(num_available)] if num_available > 0 else ["0"]
+
+        # Auto multi-GPU: if multiple GPUs and not a worker process
+        is_worker = os.environ.get("_DA3_WORKER") == "1"
+
+        if len(gpu_list) > 1 and not is_worker:
+            # Launch worker processes
+            import subprocess
+
+            num_gpus = len(gpu_list)
+            print(f"[INFO] Detected {num_gpus} GPUs: {gpu_list}")
+            print(f"[INFO] Launching {num_gpus} workers...")
+
+            # Build base command
+            base_cmd = [sys.executable, "-m", "depth_anything_3.bench.evaluator"]
+            # Pass config via dotlist instead of CLI args
+            if config_path != _default_config:
+                base_cmd += ["--config", config_path]
+            base_cmd += [f"model.path={model_path}"]
+            base_cmd += [f"workspace.work_dir={work_dir}"]
+            base_cmd += [f"eval.datasets=[{','.join(datasets)}]"]
+            base_cmd += [f"eval.modes=[{','.join(modes)}]"]
+            if scenes:
+                base_cmd += [f"eval.scenes=[{','.join(scenes)}]"]
+            base_cmd += [f"eval.max_frames={max_frames}"]
+            base_cmd += [f"eval.ref_view_strategy={ref_view_strategy}"]
+            base_cmd += [f"inference.debug={str(debug).lower()}"]
+            base_cmd += [f"inference.num_fusion_workers={num_fusion_workers}"]
+
+            # Launch workers
+            processes = []
+            for idx, gpu_id in enumerate(gpu_list):
+                env = os.environ.copy()
+                env["CUDA_VISIBLE_DEVICES"] = gpu_id
+                env["_DA3_WORKER"] = "1"  # Mark as worker process
+
+                cmd = base_cmd.copy()
+                # GPU-specific worker config
+                cmd += [f"gpu_id={idx}", f"total_gpus={num_gpus}"]
+
+                print(f"[INFO] Starting worker {idx} on GPU {gpu_id}")
+                p = subprocess.Popen(cmd, env=env)
+                processes.append(p)
+
+            # Wait for all workers
+            for p in processes:
+                p.wait()
+
+            print(f"[INFO] All {num_gpus} workers completed")
+
+            # Run evaluation after all inference is done
+            metrics = evaluator.eval()
+            evaluator.print_metrics(metrics)
+        else:
+            # Single GPU or worker process
+            from depth_anything_3.api import DepthAnything3
+
+            device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+            api = DepthAnything3.from_pretrained(model_path)
+            api = api.to(device)
+
+            evaluator.infer(api, model_path=model_path)
+
+            # Only run eval if single GPU mode (workers don't eval)
+            if not is_worker:
+                metrics = evaluator.eval()
+                evaluator.print_metrics(metrics)
+

depth_anything_3/bench/print_metrics.py

@@ -0,0 +1,618 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Beautiful metrics printing utilities for benchmark evaluation.
+
+Provides colorized, well-formatted tabular output for evaluation results.
+Supports highlighting best/worst values and grouping by dataset/mode.
+"""
+
+import argparse
+import json
+import os
+import re
+from typing import Dict as TDict, List, Optional
+
+
+# ANSI color codes for terminal output
+class Colors:
+    """ANSI escape codes for terminal colors."""
+
+    RESET = "\033[0m"
+    BOLD = "\033[1m"
+    RED = "\033[31m"
+    GREEN = "\033[32m"
+    YELLOW = "\033[33m"
+    BLUE = "\033[34m"
+    MAGENTA = "\033[35m"
+    CYAN = "\033[36m"
+    WHITE = "\033[37m"
+
+    # Bold variants
+    BOLD_RED = "\033[1;31m"
+    BOLD_GREEN = "\033[1;32m"
+    BOLD_YELLOW = "\033[1;33m"
+    BOLD_BLUE = "\033[1;34m"
+    BOLD_MAGENTA = "\033[1;35m"
+    BOLD_CYAN = "\033[1;36m"
+
+    # Background
+    BG_DARK = "\033[48;5;236m"
+
+
+def strip_ansi(text: str) -> str:
+    """Remove ANSI escape sequences from string for length calculation."""
+    ansi_escape = re.compile(r"\x1b\[[0-9;]*m")
+    return ansi_escape.sub("", text)
+
+
+def colorize_value(
+    value: str,
+    is_best: bool = False,
+    is_worst: bool = False,
+    lower_is_better: bool = False,
+) -> str:
+    """
+    Apply color to a metric value based on whether it's best/worst.
+
+    Args:
+        value: String representation of the value
+        is_best: Whether this is the best value in its column
+        is_worst: Whether this is the worst value in its column
+        lower_is_better: If True, lower values are better (e.g., error metrics)
+
+    Returns:
+        Colorized string
+    """
+    if lower_is_better:
+        # For metrics like error/distance, lower is better
+        if is_best:
+            return f"{Colors.BOLD_GREEN}{value}{Colors.RESET}"
+        elif is_worst:
+            return f"{Colors.BOLD_RED}{value}{Colors.RESET}"
+    else:
+        # For metrics like accuracy/AUC, higher is better
+        if is_best:
+            return f"{Colors.BOLD_GREEN}{value}{Colors.RESET}"
+        elif is_worst:
+            return f"{Colors.BOLD_RED}{value}{Colors.RESET}"
+    return value
+
+
+class MetricsPrinter:
+    """
+    Beautiful tabular metrics printer with color support.
+
+    Features:
+    - Colorized best/worst values
+    - Grouped by dataset and evaluation mode
+    - Automatic column width calculation
+    - Support for multiple input directories comparison
+    """
+
+    # Metrics where lower values are better
+    LOWER_IS_BETTER = {"comp", "acc", "overall", "error", "loss", "rmse", "mae"}
+
+    def __init__(self, use_color: bool = True):
+        """
+        Initialize the printer.
+
+        Args:
+            use_color: Whether to use ANSI colors in output
+        """
+        self.use_color = use_color
+
+    def print_results(self, metrics: TDict[str, dict], summary_only: bool = True) -> None:
+        """
+        Print evaluation metrics in a beautiful tabular format.
+
+        Args:
+            metrics: Dictionary mapping "dataset_mode" to metric results
+            summary_only: If True, only print summary table. If False, print per-dataset details too.
+        """
+        if not metrics:
+            print(f"\n{Colors.BOLD_RED}❌ No evaluation metrics found{Colors.RESET}")
+            return
+
+        if not summary_only:
+            self._print_header()
+            grouped = self._group_by_dataset(metrics)
+
+            for dataset, modes_data in grouped.items():
+                self._print_dataset_section(dataset, modes_data)
+
+        # Print summary table with average metrics across datasets
+        self._print_summary(metrics)
+
+        self._print_footer()
+
+    def print_comparison(
+        self,
+        metrics_list: List[TDict[str, dict]],
+        labels: List[str],
+    ) -> None:
+        """
+        Print comparison table for multiple evaluation runs.
+
+        Args:
+            metrics_list: List of metrics dictionaries
+            labels: Labels for each metrics dictionary
+        """
+        if not metrics_list or not all(metrics_list):
+            print(f"\n{Colors.BOLD_RED}❌ No metrics to compare{Colors.RESET}")
+            return
+
+        # Collect all datasets and modes
+        all_keys = set()
+        for metrics in metrics_list:
+            all_keys.update(metrics.keys())
+
+        self._print_header("COMPARISON")
+
+        for key in sorted(all_keys):
+            parts = key.rsplit("_", 1)
+            if len(parts) == 2:
+                dataset, mode = parts[0], parts[1]
+            else:
+                dataset, mode = key, "unknown"
+
+            print(f"\n{Colors.BOLD_CYAN}📊 {dataset.upper()} - {mode.upper()}{Colors.RESET}")
+            print("-" * 100)
+
+            # Collect metrics from all runs
+            all_metric_names = set()
+            for metrics in metrics_list:
+                if key in metrics and "mean" in metrics[key]:
+                    all_metric_names.update(metrics[key]["mean"].keys())
+
+            if not all_metric_names:
+                continue
+
+            # Build comparison table
+            metric_width = max(15, max(len(m) for m in all_metric_names) + 2)
+            label_width = max(15, max(len(l) for l in labels) + 2)
+
+            # Header
+            header = f"{'Metric':<{metric_width}}"
+            for label in labels:
+                header += f"{label:<{label_width}}"
+            print(header)
+            print("-" * len(strip_ansi(header)))
+
+            # Collect values for highlighting
+            for metric_name in sorted(all_metric_names):
+                values = []
+                for metrics in metrics_list:
+                    if key in metrics and "mean" in metrics[key]:
+                        val = metrics[key]["mean"].get(metric_name)
+                        values.append(val if val is not None else float("nan"))
+                    else:
+                        values.append(float("nan"))
+
+                # Find best/worst
+                valid_values = [v for v in values if not (v != v)]  # Filter NaN
+                if valid_values:
+                    lower_better = any(
+                        lb in metric_name.lower() for lb in self.LOWER_IS_BETTER
+                    )
+                    best_val = min(valid_values) if lower_better else max(valid_values)
+                    worst_val = max(valid_values) if lower_better else min(valid_values)
+                else:
+                    best_val = worst_val = None
+
+                # Print row
+                row = f"{metric_name:<{metric_width}}"
+                for val in values:
+                    if val != val:  # NaN check
+                        val_str = "N/A"
+                    else:
+                        val_str = f"{val:.4f}"
+                        if self.use_color and len(valid_values) > 1:
+                            lower_better = any(
+                                lb in metric_name.lower() for lb in self.LOWER_IS_BETTER
+                            )
+                            is_best = abs(val - best_val) < 1e-8 if best_val else False
+                            is_worst = abs(val - worst_val) < 1e-8 if worst_val else False
+                            val_str_padded = f"{val_str:<{label_width}}"
+                            val_str = colorize_value(
+                                val_str_padded, is_best, is_worst, lower_better
+                            )
+                            row += val_str
+                            continue
+                    row += f"{val_str:<{label_width}}"
+                print(row)
+
+        self._print_footer()
+
+    def _print_header(self, title: str = "EVALUATION RESULTS") -> None:
+        """Print report header."""
+        width = 100
+        print()
+        print("=" * width)
+        print(f"{Colors.BOLD_CYAN}📊 DEPTH ANYTHING 3 {title}{Colors.RESET}")
+        print("=" * width)
+
+    def _print_footer(self) -> None:
+        """Print report footer."""
+        width = 100
+        print()
+        print("=" * width)
+        print(f"{Colors.BOLD_GREEN}✅ Evaluation Complete{Colors.RESET}")
+        print("=" * width)
+        print()
+
+    def _group_by_dataset(self, metrics: TDict[str, dict]) -> TDict[str, dict]:
+        """Group metrics by dataset."""
+        grouped = {}
+        for key, data in metrics.items():
+            if not isinstance(data, dict) or "mean" not in data:
+                continue
+            # Parse key format: "dataset_mode" (e.g., "dtu_recon_unposed")
+            parts = key.split("_", 1)
+            if len(parts) == 2:
+                dataset, mode = parts
+                if dataset not in grouped:
+                    grouped[dataset] = {}
+                grouped[dataset][mode] = data
+        return grouped
+
+    def _print_dataset_section(self, dataset: str, modes_data: TDict[str, dict]) -> None:
+        """Print metrics section for a single dataset."""
+        print(f"\n{Colors.BOLD_MAGENTA}🔍 {dataset.upper()}{Colors.RESET}")
+        print("-" * 100)
+
+        # Collect all unique metrics across all modes
+        all_metrics = set()
+        for mode_data in modes_data.values():
+            all_metrics.update(mode_data["mean"].keys())
+        all_metrics = sorted(list(all_metrics))
+
+        if not all_metrics:
+            print("  No metrics available")
+            return
+
+        # Calculate column widths
+        metric_width = max(18, max(len(m) for m in all_metrics) + 2)
+        mode_width = 18
+        modes = list(modes_data.keys())
+
+        # Print header
+        header = f"{'Metric':<{metric_width}}"
+        for mode in modes:
+            header += f"{mode.upper():<{mode_width}}"
+        print(f"{Colors.BOLD}{header}{Colors.RESET}")
+        print("-" * len(header))
+
+        # Print each metric row
+        for metric in all_metrics:
+            row = f"{metric:<{metric_width}}"
+
+            # Collect values for this metric across modes
+            values = []
+            for mode in modes:
+                if metric in modes_data[mode]["mean"]:
+                    values.append(modes_data[mode]["mean"][metric])
+                else:
+                    values.append(None)
+
+            # Find best/worst values
+            valid_values = [v for v in values if v is not None]
+            if valid_values:
+                lower_better = any(lb in metric.lower() for lb in self.LOWER_IS_BETTER)
+                best_val = min(valid_values) if lower_better else max(valid_values)
+                worst_val = max(valid_values) if lower_better else min(valid_values)
+            else:
+                best_val = worst_val = None
+
+            # Format each value
+            for val in values:
+                if val is None:
+                    row += f"{'N/A':<{mode_width}}"
+                else:
+                    val_str = f"{val:.4f}"
+                    if self.use_color and len(valid_values) > 1:
+                        is_best = abs(val - best_val) < 1e-8 if best_val else False
+                        is_worst = abs(val - worst_val) < 1e-8 if worst_val else False
+                        lower_better = any(
+                            lb in metric.lower() for lb in self.LOWER_IS_BETTER
+                        )
+                        # Pad before colorizing to maintain alignment
+                        val_str_padded = f"{val_str:<{mode_width}}"
+                        row += colorize_value(
+                            val_str_padded, is_best, is_worst, lower_better
+                        )
+                    else:
+                        row += f"{val_str:<{mode_width}}"
+            print(row)
+
+        # Show scene counts
+        scene_info = []
+        for mode, mode_data in modes_data.items():
+            scene_count = len([k for k in mode_data.keys() if k != "mean"])
+            scene_info.append(f"{mode}: {scene_count} scenes")
+        print(f"\n{Colors.CYAN}📈 {' | '.join(scene_info)}{Colors.RESET}")
+
+    def _print_summary(self, metrics: TDict[str, dict]) -> None:
+        """
+        Print summary table with key metrics across all datasets.
+
+        Format: One row per metric, datasets as columns.
+        Order: HiRoom, ETH3D, DTU, 7Scenes, ScanNet++, (DTU-64 for pose only)
+        """
+        print(f"\n{Colors.BOLD_CYAN}{'=' * 120}{Colors.RESET}")
+        print(f"{Colors.BOLD_CYAN}📊 SUMMARY{Colors.RESET}")
+        print(f"{Colors.BOLD_CYAN}{'=' * 120}{Colors.RESET}")
+
+        # Dataset display order and names
+        DATASET_ORDER = ["hiroom", "eth3d", "dtu", "7scenes", "scannetpp", "dtu64"]
+        DATASET_DISPLAY = {
+            "hiroom": "HiRoom",
+            "eth3d": "ETH3D",
+            "dtu": "DTU",
+            "7scenes": "7Scenes",
+            "scannetpp": "ScanNet++",
+            "dtu64": "DTU-64",
+        }
+
+        # Collect all metrics into a structured dict
+        # metric_data[dataset][mode] = {"Auc_3": x, "Auc_30": x, "fscore": x, "overall": x}
+        metric_data = {}
+        for key, data in metrics.items():
+            if not isinstance(data, dict) or "mean" not in data:
+                continue
+            parts = key.split("_", 1)
+            if len(parts) != 2:
+                continue
+            dataset, mode = parts
+            dataset_lower = dataset.lower()
+            if dataset_lower not in metric_data:
+                metric_data[dataset_lower] = {}
+            metric_data[dataset_lower][mode] = data["mean"]
+
+        col_width = 12
+
+        def fmt_val(val):
+            """Format value or return N/A."""
+            if val is None:
+                return "N/A"
+            return f"{val:.4f}"
+
+        def get_metric(dataset, mode, metric_name):
+            """Get metric value or None."""
+            if dataset not in metric_data:
+                return None
+            if mode not in metric_data[dataset]:
+                return None
+            return metric_data[dataset][mode].get(metric_name)
+
+        # ============ POSE METRICS ============
+        print(f"\n{Colors.BOLD_MAGENTA}🎯 POSE ESTIMATION{Colors.RESET}")
+        
+        # Pose: show all datasets except DTU (keep DTU-64 only)
+        # Order: HiRoom, ETH3D, DTU-64, 7Scenes, ScanNet++
+        pose_datasets = ["hiroom", "eth3d", "dtu64", "7scenes", "scannetpp"]
+        
+        # Header: Avg first, then datasets
+        header = f"{'Metric':<15}{'Avg':<{col_width}}"
+        for ds in pose_datasets:
+            header += f"{DATASET_DISPLAY[ds]:<{col_width}}"
+        print("-" * len(strip_ansi(header)))
+        print(f"{Colors.BOLD}{header}{Colors.RESET}")
+        print("-" * len(strip_ansi(header)))
+
+        # Helper to get metric with fallback names
+        def get_pose_metric(dataset, metric_name):
+            """Get pose metric with fallback for different naming conventions."""
+            # Try different naming conventions
+            names = {
+                "Auc3": ["Auc_3", "auc03", "auc_3", "AUC_3", "Auc3", "auc3"],
+                "Auc30": ["Auc_30", "auc30", "auc_30", "AUC_30", "Auc30"],
+            }
+            for name in names.get(metric_name, [metric_name]):
+                val = get_metric(dataset, "pose", name)
+                if val is not None:
+                    return val
+            return None
+
+        # Auc3 row
+        values = []
+        for ds in pose_datasets:
+            val = get_pose_metric(ds, "Auc3")
+            if val is not None:
+                values.append(val)
+        avg = sum(values) / len(values) if values else None
+        row = f"{'Auc3':<15}{Colors.BOLD_GREEN}{fmt_val(avg):<{col_width}}{Colors.RESET}"
+        for ds in pose_datasets:
+            val = get_pose_metric(ds, "Auc3")
+            row += f"{fmt_val(val):<{col_width}}"
+        print(row)
+
+        # Auc30 row
+        values = []
+        for ds in pose_datasets:
+            val = get_pose_metric(ds, "Auc30")
+            if val is not None:
+                values.append(val)
+        avg = sum(values) / len(values) if values else None
+        row = f"{'Auc30':<15}{Colors.BOLD_GREEN}{fmt_val(avg):<{col_width}}{Colors.RESET}"
+        for ds in pose_datasets:
+            val = get_pose_metric(ds, "Auc30")
+            row += f"{fmt_val(val):<{col_width}}"
+        print(row)
+
+        # ============ RECON_UNPOSED METRICS ============
+        print(f"\n{Colors.BOLD_MAGENTA}🏗️  RECON_UNPOSED (Pred Pose){Colors.RESET}")
+        
+        # For recon, exclude dtu64 from columns
+        recon_datasets = ["hiroom", "eth3d", "dtu", "7scenes", "scannetpp"]
+        avg_datasets = ["hiroom", "eth3d", "7scenes", "scannetpp"]  # Exclude DTU from avg
+        
+        # Header: Avg first, then datasets
+        header = f"{'Metric':<15}{'Avg*':<{col_width}}"
+        for ds in recon_datasets:
+            header += f"{DATASET_DISPLAY[ds]:<{col_width}}"
+        print("-" * len(strip_ansi(header)))
+        print(f"{Colors.BOLD}{header}{Colors.RESET}")
+        print("-" * len(strip_ansi(header)))
+
+        # F-score row (only metric for avg)
+        values = []
+        for ds in recon_datasets:
+            val = get_metric(ds, "recon_unposed", "fscore")
+            if val is not None and ds in avg_datasets:
+                values.append(val)
+        avg = sum(values) / len(values) if values else None
+        row = f"{'F-score':<15}{Colors.BOLD_GREEN}{fmt_val(avg):<{col_width}}{Colors.RESET}"
+        for ds in recon_datasets:
+            val = get_metric(ds, "recon_unposed", "fscore")
+            row += f"{fmt_val(val):<{col_width}}"
+        print(row)
+
+        # Overall row (avg over 4 datasets excluding DTU)
+        values = []
+        for ds in recon_datasets:
+            val = get_metric(ds, "recon_unposed", "overall")
+            if val is not None and ds in avg_datasets:
+                values.append(val)
+        avg = sum(values) / len(values) if values else None
+        row = f"{'Overall':<15}{Colors.BOLD_GREEN}{fmt_val(avg):<{col_width}}{Colors.RESET}"
+        for ds in recon_datasets:
+            val = get_metric(ds, "recon_unposed", "overall")
+            row += f"{fmt_val(val):<{col_width}}"
+        print(row)
+
+        # ============ RECON_POSED METRICS ============
+        print(f"\n{Colors.BOLD_MAGENTA}🏗️  RECON_POSED (GT Pose){Colors.RESET}")
+        
+        # Header: Avg first, then datasets
+        header = f"{'Metric':<15}{'Avg*':<{col_width}}"
+        for ds in recon_datasets:
+            header += f"{DATASET_DISPLAY[ds]:<{col_width}}"
+        print("-" * len(strip_ansi(header)))
+        print(f"{Colors.BOLD}{header}{Colors.RESET}")
+        print("-" * len(strip_ansi(header)))
+
+        # F-score row (only metric for avg)
+        values = []
+        for ds in recon_datasets:
+            val = get_metric(ds, "recon_posed", "fscore")
+            if val is not None and ds in avg_datasets:
+                values.append(val)
+        avg = sum(values) / len(values) if values else None
+        row = f"{'F-score':<15}{Colors.BOLD_GREEN}{fmt_val(avg):<{col_width}}{Colors.RESET}"
+        for ds in recon_datasets:
+            val = get_metric(ds, "recon_posed", "fscore")
+            row += f"{fmt_val(val):<{col_width}}"
+        print(row)
+
+        # Overall row (avg over 4 datasets excluding DTU)
+        values = []
+        for ds in recon_datasets:
+            val = get_metric(ds, "recon_posed", "overall")
+            if val is not None and ds in avg_datasets:
+                values.append(val)
+        avg = sum(values) / len(values) if values else None
+        row = f"{'Overall':<15}{Colors.BOLD_GREEN}{fmt_val(avg):<{col_width}}{Colors.RESET}"
+        for ds in recon_datasets:
+            val = get_metric(ds, "recon_posed", "overall")
+            row += f"{fmt_val(val):<{col_width}}"
+        print(row)
+
+        print(f"\n{Colors.CYAN}* Avg F-score / Overall = average over HiRoom, ETH3D, 7Scenes, ScanNet++ (4 datasets){Colors.RESET}")
+
+
+def load_metrics_from_dir(metric_dir: str) -> TDict[str, dict]:
+    """
+    Load all metrics JSON files from a directory.
+
+    Args:
+        metric_dir: Path to directory containing metric JSON files
+
+    Returns:
+        Dictionary mapping filename (without .json) to metric data
+    """
+    metrics = {}
+    if not os.path.exists(metric_dir):
+        return metrics
+
+    for filename in os.listdir(metric_dir):
+        if filename.endswith(".json"):
+            filepath = os.path.join(metric_dir, filename)
+            try:
+                with open(filepath, encoding="utf-8") as f:
+                    content = f.read()
+                # Handle trailing commas in JSON
+                content = re.sub(r",\s*([\]\}])", r"\1", content)
+                data = json.loads(content)
+                key = filename[:-5]
+                metrics[key] = data
+            except Exception as e:
+                print(f"Warning: Failed to load {filename}: {e}")
+
+    return metrics
+
+
+def main():
+    """Command-line interface for metrics printing."""
+    parser = argparse.ArgumentParser(
+        description="Print DepthAnything3 benchmark evaluation metrics."
+    )
+    parser.add_argument(
+        "--input_dir",
+        type=str,
+        default="./eval_workspace/metric_results",
+        help="Directory containing metric JSON files (comma-separated for comparison)",
+    )
+    parser.add_argument(
+        "--no_color",
+        action="store_true",
+        help="Disable colored output",
+    )
+    parser.add_argument(
+        "--key",
+        type=str,
+        default=None,
+        help="Specific metric key to highlight",
+    )
+    args = parser.parse_args()
+
+    # Support multiple directories for comparison
+    input_dirs = [d.strip() for d in args.input_dir.split(",") if d.strip()]
+
+    printer = MetricsPrinter(use_color=not args.no_color)
+
+    if len(input_dirs) == 1:
+        # Single directory - simple print
+        metrics = load_metrics_from_dir(input_dirs[0])
+        printer.print_results(metrics)
+    else:
+        # Multiple directories - comparison mode
+        metrics_list = []
+        labels = []
+        for d in input_dirs:
+            metrics = load_metrics_from_dir(d)
+            if metrics:
+                metrics_list.append(metrics)
+                labels.append(os.path.basename(d.rstrip("/")))
+
+        if metrics_list:
+            printer.print_comparison(metrics_list, labels)
+        else:
+            print("No metrics found in specified directories")
+
+
+if __name__ == "__main__":
+    main()
+

depth_anything_3/bench/registries.py

@@ -0,0 +1,85 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Auto-loading registry system for benchmark datasets.
+
+This module provides registry classes that automatically discover and import
+dataset implementations from the datasets subpackage on first access.
+"""
+
+import importlib
+import pkgutil
+import threading
+
+from depth_anything_3.utils.registry import Registry
+
+__all__ = ["METRIC_REGISTRY", "MONO_REGISTRY", "MV_REGISTRY", "NVS_REGISTRY"]
+
+# ---- Lazy import: Only scan and import all datasets submodules on first registry access ----
+_loaded = False
+_lock = threading.Lock()
+
+
+def _import_all_datasets_once():
+    """
+    Scan and import all .py submodules under depth_anything_3.bench.datasets
+    (skip files/packages starting with underscore), to trigger @REGISTRY.register(...) in each module.
+    """
+    global _loaded
+    if _loaded:
+        return
+
+    with _lock:
+        if _loaded:
+            return
+
+        pkg_name = "depth_anything_3.bench.datasets"
+        pkg = importlib.import_module(pkg_name)
+        pkg_paths = list(getattr(pkg, "__path__", []))
+
+        for finder, name, ispkg in pkgutil.walk_packages(pkg_paths, prefix=pkg_name + "."):
+            base = name.rsplit(".", 1)[-1]
+            if base.startswith("_"):
+                continue
+            try:
+                importlib.import_module(name)
+            except Exception as e:
+                print(f"[datasets auto-import] Failed to import {name}: {e}")
+
+        _loaded = True
+
+
+class AutoRegistry(Registry):
+    """Registry that ensures all datasets are auto-discovered and imported on first use."""
+
+    def get(self, name):
+        _import_all_datasets_once()
+        return super().get(name)
+
+    def all(self):
+        _import_all_datasets_once()
+        return super().all()
+
+    def has(self, name):
+        _import_all_datasets_once()
+        return name in self._map
+
+
+# Four auto-lazy registry instances for different evaluation types
+METRIC_REGISTRY = AutoRegistry()  # For metric depth evaluation
+MONO_REGISTRY = AutoRegistry()  # For monocular depth evaluation
+MV_REGISTRY = AutoRegistry()  # For multi-view evaluation
+NVS_REGISTRY = AutoRegistry()  # For novel view synthesis evaluation
+

depth_anything_3/bench/utils.py

@@ -0,0 +1,525 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Utility functions for benchmark evaluation.
+
+Contains:
+- Pose evaluation metrics (AUC) and helper functions
+- 3D reconstruction evaluation metrics (Acc/Comp/F-score)
+- Geometry utilities (quaternion conversion, etc.)
+"""
+
+from typing import Dict as TDict, Optional, Tuple, Union
+
+import numpy as np
+import open3d as o3d
+import torch
+from addict import Dict
+from scipy.spatial import KDTree
+
+from depth_anything_3.utils.geometry import mat_to_quat
+
+
+# =============================================================================
+# Geometry Utilities
+# =============================================================================
+
+
+def quat2rotmat(qvec: list) -> np.ndarray:
+    """
+    Convert quaternion (WXYZ order) to rotation matrix.
+
+    Args:
+        qvec: Quaternion as [w, x, y, z]
+
+    Returns:
+        3x3 rotation matrix
+    """
+    rotmat = np.array(
+        [
+            1 - 2 * qvec[2] ** 2 - 2 * qvec[3] ** 2,
+            2 * qvec[1] * qvec[2] - 2 * qvec[0] * qvec[3],
+            2 * qvec[3] * qvec[1] + 2 * qvec[0] * qvec[2],
+            2 * qvec[1] * qvec[2] + 2 * qvec[0] * qvec[3],
+            1 - 2 * qvec[1] ** 2 - 2 * qvec[3] ** 2,
+            2 * qvec[2] * qvec[3] - 2 * qvec[0] * qvec[1],
+            2 * qvec[3] * qvec[1] - 2 * qvec[0] * qvec[2],
+            2 * qvec[2] * qvec[3] + 2 * qvec[0] * qvec[1],
+            1 - 2 * qvec[1] ** 2 - 2 * qvec[2] ** 2,
+        ]
+    )
+    rotmat = rotmat.reshape(3, 3)
+    return rotmat
+
+
+# =============================================================================
+# 3D Reconstruction Evaluation
+# =============================================================================
+
+
+def nn_correspondance(verts1: np.ndarray, verts2: np.ndarray) -> np.ndarray:
+    """
+    Compute nearest neighbor distances from verts2 to verts1 using KDTree.
+
+    Args:
+        verts1: Reference point cloud [N, 3]
+        verts2: Query point cloud [M, 3]
+
+    Returns:
+        Distance array [M,] - distance from each point in verts2 to nearest in verts1
+    """
+    if len(verts1) == 0 or len(verts2) == 0:
+        return np.array([])
+
+    kdtree = KDTree(verts1)
+    distances, _ = kdtree.query(verts2)
+    return distances.reshape(-1)
+
+
+def evaluate_3d_reconstruction(
+    pcd_pred: Union[o3d.geometry.PointCloud, np.ndarray],
+    pcd_trgt: Union[o3d.geometry.PointCloud, np.ndarray],
+    threshold: float = 0.05,
+    down_sample: Optional[float] = None,
+) -> TDict[str, float]:
+    """
+    Evaluate 3D reconstruction quality using standard metrics.
+
+    This function computes:
+    - Accuracy: Mean distance from predicted points to GT surface
+    - Completeness: Mean distance from GT points to predicted surface
+    - Overall: Average of accuracy and completeness
+    - Precision: Fraction of predicted points within threshold of GT
+    - Recall: Fraction of GT points within threshold of prediction
+    - F-score: Harmonic mean of precision and recall
+
+    Args:
+        pcd_pred: Predicted point cloud (Open3D or numpy array)
+        pcd_trgt: Ground truth point cloud (Open3D or numpy array)
+        threshold: Distance threshold for precision/recall (meters)
+        down_sample: Voxel size for downsampling (None to skip)
+
+    Returns:
+        Dict with metrics: acc, comp, overall, precision, recall, fscore
+    """
+    # Convert to Open3D if needed
+    if isinstance(pcd_pred, np.ndarray):
+        pcd_pred_o3d = o3d.geometry.PointCloud()
+        pcd_pred_o3d.points = o3d.utility.Vector3dVector(pcd_pred)
+        pcd_pred = pcd_pred_o3d
+    if isinstance(pcd_trgt, np.ndarray):
+        pcd_trgt_o3d = o3d.geometry.PointCloud()
+        pcd_trgt_o3d.points = o3d.utility.Vector3dVector(pcd_trgt)
+        pcd_trgt = pcd_trgt_o3d
+
+    # Downsample if requested
+    if down_sample is not None and down_sample > 0:
+        pcd_pred = pcd_pred.voxel_down_sample(down_sample)
+        pcd_trgt = pcd_trgt.voxel_down_sample(down_sample)
+
+    verts_pred = np.asarray(pcd_pred.points)
+    verts_trgt = np.asarray(pcd_trgt.points)
+
+    # Handle empty point clouds
+    if len(verts_pred) == 0 or len(verts_trgt) == 0:
+        return {
+            "acc": float("inf"),
+            "comp": float("inf"),
+            "overall": float("inf"),
+            "precision": 0.0,
+            "recall": 0.0,
+            "fscore": 0.0,
+        }
+
+    # Compute distances
+    dist_pred_to_gt = nn_correspondance(verts_trgt, verts_pred)  # Accuracy
+    dist_gt_to_pred = nn_correspondance(verts_pred, verts_trgt)  # Completeness
+
+    # Compute metrics
+    accuracy = float(np.mean(dist_pred_to_gt))
+    completeness = float(np.mean(dist_gt_to_pred))
+    overall = (accuracy + completeness) / 2
+
+    precision = float(np.mean((dist_pred_to_gt < threshold).astype(float)))
+    recall = float(np.mean((dist_gt_to_pred < threshold).astype(float)))
+
+    if precision + recall > 0:
+        fscore = 2 * precision * recall / (precision + recall)
+    else:
+        fscore = 0.0
+
+    return {
+        "acc": accuracy,
+        "comp": completeness,
+        "overall": overall,
+        "precision": precision,
+        "recall": recall,
+        "fscore": fscore,
+    }
+
+
+def create_tsdf_volume(
+    voxel_length: float = 4.0 / 512.0,
+    sdf_trunc: float = 0.04,
+    color_type: str = "RGB8",
+) -> o3d.pipelines.integration.ScalableTSDFVolume:
+    """
+    Create a scalable TSDF volume for depth fusion.
+
+    Args:
+        voxel_length: Size of each voxel
+        sdf_trunc: Truncation distance for SDF
+        color_type: Color integration type ("RGB8" or "Gray32")
+
+    Returns:
+        Initialized ScalableTSDFVolume
+    """
+    if color_type == "RGB8":
+        color_enum = o3d.pipelines.integration.TSDFVolumeColorType.RGB8
+    else:
+        color_enum = o3d.pipelines.integration.TSDFVolumeColorType.Gray32
+
+    volume = o3d.pipelines.integration.ScalableTSDFVolume(
+        voxel_length=voxel_length,
+        sdf_trunc=sdf_trunc,
+        color_type=color_enum,
+    )
+    return volume
+
+
+def fuse_depth_to_tsdf(
+    volume: o3d.pipelines.integration.ScalableTSDFVolume,
+    depths: np.ndarray,
+    images: np.ndarray,
+    intrinsics: np.ndarray,
+    extrinsics: np.ndarray,
+    max_depth: float = 10.0,
+) -> o3d.geometry.TriangleMesh:
+    """
+    Fuse multiple depth maps into TSDF volume and extract mesh.
+
+    Args:
+        volume: TSDF volume to integrate into
+        depths: Depth maps [N, H, W]
+        images: RGB images [N, H, W, 3]
+        intrinsics: Camera intrinsics [N, 3, 3]
+        extrinsics: Camera extrinsics (world-to-camera) [N, 4, 4]
+        max_depth: Maximum depth for truncation
+
+    Returns:
+        Extracted triangle mesh
+    """
+    for i in range(len(depths)):
+        depth = depths[i]
+        image = images[i]
+        ixt = intrinsics[i]
+        ext = extrinsics[i]
+
+        h, w = depth.shape[:2]
+
+        # Create RGBD image
+        depth_o3d = o3d.geometry.Image(depth.astype(np.float32))
+        color_o3d = o3d.geometry.Image(image.astype(np.uint8))
+        rgbd = o3d.geometry.RGBDImage.create_from_color_and_depth(
+            color_o3d,
+            depth_o3d,
+            depth_trunc=max_depth,
+            convert_rgb_to_intensity=False,
+            depth_scale=1.0,
+        )
+
+        # Create camera intrinsics
+        ixt_o3d = o3d.camera.PinholeCameraIntrinsic(
+            w, h, ixt[0, 0], ixt[1, 1], ixt[0, 2], ixt[1, 2]
+        )
+
+        # Integrate into volume
+        volume.integrate(rgbd, ixt_o3d, ext)
+
+    # Extract mesh
+    mesh = volume.extract_triangle_mesh()
+    return mesh
+
+
+def sample_points_from_mesh(
+    mesh: o3d.geometry.TriangleMesh,
+    num_points: int = 1000000,
+) -> o3d.geometry.PointCloud:
+    """
+    Uniformly sample points from a triangle mesh.
+
+    Args:
+        mesh: Input triangle mesh
+        num_points: Number of points to sample
+
+    Returns:
+        Sampled point cloud
+    """
+    try:
+        pcd = mesh.sample_points_uniformly(number_of_points=num_points)
+        # Clamp colors to valid range [0, 1] for Open3D PLY export
+        if pcd.has_colors():
+            colors = np.asarray(pcd.colors)
+            colors = np.clip(colors, 0.0, 1.0)
+            pcd.colors = o3d.utility.Vector3dVector(colors)
+    except Exception:
+        # Fallback: create random points if mesh is invalid (with fixed seed for reproducibility)
+        rng = np.random.default_rng(seed=42)
+        points = rng.uniform(-1, 1, size=(num_points, 3))
+        pcd = o3d.geometry.PointCloud()
+        pcd.points = o3d.utility.Vector3dVector(points)
+    return pcd
+
+
+# =============================================================================
+# Pose Evaluation
+# =============================================================================
+
+
+def build_pair_index(N: int, B: int = 1):
+    """
+    Build indices for all possible pairs of frames.
+
+    Args:
+        N: Number of frames
+        B: Batch size
+
+    Returns:
+        i1, i2: Indices for all possible pairs
+    """
+    i1_, i2_ = torch.combinations(torch.arange(N), 2, with_replacement=False).unbind(-1)
+    i1, i2 = ((i[None] + torch.arange(B)[:, None] * N).reshape(-1) for i in [i1_, i2_])
+    return i1, i2
+
+
+def compute_pose(pred_se3: torch.Tensor, gt_se3: torch.Tensor) -> Dict:
+    """
+    Compute pose estimation metrics between predicted and ground truth trajectories.
+
+    Args:
+        pred_se3: Predicted SE(3) transformations [N, 4, 4]
+        gt_se3: Ground truth SE(3) transformations [N, 4, 4]
+
+    Returns:
+        Dict with AUC metrics at different thresholds (auc30, auc15, auc05, auc03)
+    """
+    pred_se3 = align_to_first_camera(pred_se3)
+    gt_se3 = align_to_first_camera(gt_se3)
+
+    rel_rangle_deg, rel_tangle_deg = se3_to_relative_pose_error(pred_se3, gt_se3, len(pred_se3))
+    rError = rel_rangle_deg.cpu().numpy()
+    tError = rel_tangle_deg.cpu().numpy()
+
+    output = Dict()
+    output.auc30, _ = calculate_auc_np(rError, tError, max_threshold=30)
+    output.auc15, _ = calculate_auc_np(rError, tError, max_threshold=15)
+    output.auc05, _ = calculate_auc_np(rError, tError, max_threshold=5)
+    output.auc03, _ = calculate_auc_np(rError, tError, max_threshold=3)
+    return output
+
+
+def align_to_first_camera(camera_poses: torch.Tensor) -> torch.Tensor:
+    """
+    Align all camera poses to the first camera's coordinate frame.
+
+    Args:
+        camera_poses: Camera poses as SE3 transformations [N, 4, 4]
+
+    Returns:
+        Aligned camera poses [N, 4, 4]
+    """
+    first_cam_extrinsic_inv = closed_form_inverse_se3(camera_poses[0][None])
+    aligned_poses = torch.matmul(camera_poses, first_cam_extrinsic_inv)
+    return aligned_poses
+
+
+def rotation_angle(
+    rot_gt: torch.Tensor, rot_pred: torch.Tensor, batch_size: int = None, eps: float = 1e-15
+) -> torch.Tensor:
+    """
+    Calculate rotation angle error between ground truth and predicted rotations.
+
+    Args:
+        rot_gt: Ground truth rotation matrices
+        rot_pred: Predicted rotation matrices
+        batch_size: Batch size for reshaping the result
+        eps: Small value to avoid numerical issues
+
+    Returns:
+        Rotation angle error in degrees
+    """
+    q_pred = mat_to_quat(rot_pred)
+    q_gt = mat_to_quat(rot_gt)
+
+    loss_q = (1 - (q_pred * q_gt).sum(dim=1) ** 2).clamp(min=eps)
+    err_q = torch.arccos(1 - 2 * loss_q)
+
+    rel_rangle_deg = err_q * 180 / np.pi
+
+    if batch_size is not None:
+        rel_rangle_deg = rel_rangle_deg.reshape(batch_size, -1)
+
+    return rel_rangle_deg
+
+
+def translation_angle(
+    tvec_gt: torch.Tensor,
+    tvec_pred: torch.Tensor,
+    batch_size: int = None,
+    ambiguity: bool = True,
+) -> torch.Tensor:
+    """
+    Calculate translation angle error between ground truth and predicted translations.
+
+    Args:
+        tvec_gt: Ground truth translation vectors
+        tvec_pred: Predicted translation vectors
+        batch_size: Batch size for reshaping the result
+        ambiguity: Whether to handle direction ambiguity
+
+    Returns:
+        Translation angle error in degrees
+    """
+    rel_tangle_deg = compare_translation_by_angle(tvec_gt, tvec_pred)
+    rel_tangle_deg = rel_tangle_deg * 180.0 / np.pi
+
+    if ambiguity:
+        rel_tangle_deg = torch.min(rel_tangle_deg, (180 - rel_tangle_deg).abs())
+
+    if batch_size is not None:
+        rel_tangle_deg = rel_tangle_deg.reshape(batch_size, -1)
+
+    return rel_tangle_deg
+
+
+def compare_translation_by_angle(
+    t_gt: torch.Tensor, t: torch.Tensor, eps: float = 1e-15, default_err: float = 1e6
+) -> torch.Tensor:
+    """
+    Normalize the translation vectors and compute the angle between them.
+
+    Args:
+        t_gt: Ground truth translation vectors
+        t: Predicted translation vectors
+        eps: Small value to avoid division by zero
+        default_err: Default error value for invalid cases
+
+    Returns:
+        Angular error between translation vectors in radians
+    """
+    t_norm = torch.norm(t, dim=1, keepdim=True)
+    t = t / (t_norm + eps)
+
+    t_gt_norm = torch.norm(t_gt, dim=1, keepdim=True)
+    t_gt = t_gt / (t_gt_norm + eps)
+
+    loss_t = torch.clamp_min(1.0 - torch.sum(t * t_gt, dim=1) ** 2, eps)
+    err_t = torch.acos(torch.sqrt(1 - loss_t))
+
+    err_t[torch.isnan(err_t) | torch.isinf(err_t)] = default_err
+    return err_t
+
+
+def calculate_auc_np(
+    r_error: np.ndarray, t_error: np.ndarray, max_threshold: int = 30
+) -> tuple:
+    """
+    Calculate the Area Under the Curve (AUC) for the given error arrays.
+
+    Args:
+        r_error: Rotation error values in degrees
+        t_error: Translation error values in degrees
+        max_threshold: Maximum threshold value for binning
+
+    Returns:
+        Tuple of (AUC value, normalized histogram)
+    """
+    error_matrix = np.concatenate((r_error[:, None], t_error[:, None]), axis=1)
+    max_errors = np.max(error_matrix, axis=1)
+    bins = np.arange(max_threshold + 1)
+    histogram, _ = np.histogram(max_errors, bins=bins)
+    num_pairs = float(len(max_errors))
+    normalized_histogram = histogram.astype(float) / num_pairs
+    return np.mean(np.cumsum(normalized_histogram)), normalized_histogram
+
+
+def se3_to_relative_pose_error(
+    pred_se3: torch.Tensor, gt_se3: torch.Tensor, num_frames: int
+) -> tuple:
+    """
+    Compute rotation and translation errors between predicted and ground truth poses.
+
+    Args:
+        pred_se3: Predicted SE(3) transformations
+        gt_se3: Ground truth SE(3) transformations
+        num_frames: Number of frames
+
+    Returns:
+        Tuple of (rotation angle errors, translation angle errors) in degrees
+    """
+    pair_idx_i1, pair_idx_i2 = build_pair_index(num_frames)
+
+    # Compute relative camera poses between pairs using closed-form inverse
+    relative_pose_gt = closed_form_inverse_se3(gt_se3[pair_idx_i1]).bmm(gt_se3[pair_idx_i2])
+    relative_pose_pred = closed_form_inverse_se3(pred_se3[pair_idx_i1]).bmm(pred_se3[pair_idx_i2])
+
+    # Compute the difference in rotation and translation
+    rel_rangle_deg = rotation_angle(relative_pose_gt[:, :3, :3], relative_pose_pred[:, :3, :3])
+    rel_tangle_deg = translation_angle(relative_pose_gt[:, :3, 3], relative_pose_pred[:, :3, 3])
+
+    return rel_rangle_deg, rel_tangle_deg
+
+
+def closed_form_inverse_se3(
+    se3: torch.Tensor, R: torch.Tensor = None, T: torch.Tensor = None
+) -> torch.Tensor:
+    """
+    Compute the inverse of each 4x4 (or 3x4) SE3 matrix in a batch.
+
+    Uses closed-form solution instead of torch.inverse() for numerical stability.
+
+    Args:
+        se3: Nx4x4 or Nx3x4 tensor of SE3 matrices
+        R: Optional Nx3x3 rotation matrices
+        T: Optional Nx3x1 translation vectors
+
+    Returns:
+        Inverted SE3 matrices with same shape as input
+    """
+    is_numpy = isinstance(se3, np.ndarray)
+
+    if se3.shape[-2:] != (4, 4) and se3.shape[-2:] != (3, 4):
+        raise ValueError(f"se3 must be of shape (N,4,4), got {se3.shape}.")
+
+    if R is None:
+        R = se3[:, :3, :3]
+    if T is None:
+        T = se3[:, :3, 3:]
+
+    if is_numpy:
+        R_transposed = np.transpose(R, (0, 2, 1))
+        top_right = -np.matmul(R_transposed, T)
+        inverted_matrix = np.tile(np.eye(4), (len(R), 1, 1))
+    else:
+        R_transposed = R.transpose(1, 2)
+        top_right = -torch.bmm(R_transposed, T)
+        inverted_matrix = torch.eye(4, 4)[None].repeat(len(R), 1, 1)
+        inverted_matrix = inverted_matrix.to(R.dtype).to(R.device)
+
+    inverted_matrix[:, :3, :3] = R_transposed
+    inverted_matrix[:, :3, 3:] = top_right
+
+    return inverted_matrix
+

depth_anything_3/cfg.py

@@ -0,0 +1,144 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Configuration utility functions
+"""
+
+import importlib
+from pathlib import Path
+from typing import Any, Callable, List, Union
+from omegaconf import DictConfig, ListConfig, OmegaConf
+
+try:
+    OmegaConf.register_new_resolver("eval", eval)
+except Exception as e:
+    # if eval is not available, we can just pass
+    print(f"Error registering eval resolver: {e}")
+
+
+def load_config(path: str, argv: List[str] = None) -> Union[DictConfig, ListConfig]:
+    """
+    Load a configuration. Will resolve inheritance.
+    Supports both file paths and module paths (e.g., depth_anything_3.configs.giant).
+    """
+    # Check if path is a module path (contains dots but no slashes and doesn't end with .yaml)
+    if "." in path and "/" not in path and not path.endswith(".yaml"):
+        # It's a module path, load from package resources
+        path_parts = path.split(".")[1:]
+        config_path = Path(__file__).resolve().parent
+        for part in path_parts:
+            config_path = config_path.joinpath(part)
+        config_path = config_path.with_suffix(".yaml")
+        config = OmegaConf.load(str(config_path))
+    else:
+        # It's a file path (absolute, relative, or with .yaml extension)
+        config = OmegaConf.load(path)
+
+    if argv is not None:
+        config_argv = OmegaConf.from_dotlist(argv)
+        config = OmegaConf.merge(config, config_argv)
+    config = resolve_recursive(config, resolve_inheritance)
+    return config
+
+
+def resolve_recursive(
+    config: Any,
+    resolver: Callable[[Union[DictConfig, ListConfig]], Union[DictConfig, ListConfig]],
+) -> Any:
+    config = resolver(config)
+    if isinstance(config, DictConfig):
+        for k in config.keys():
+            v = config.get(k)
+            if isinstance(v, (DictConfig, ListConfig)):
+                config[k] = resolve_recursive(v, resolver)
+    if isinstance(config, ListConfig):
+        for i in range(len(config)):
+            v = config.get(i)
+            if isinstance(v, (DictConfig, ListConfig)):
+                config[i] = resolve_recursive(v, resolver)
+    return config
+
+
+def resolve_inheritance(config: Union[DictConfig, ListConfig]) -> Any:
+    """
+    Recursively resolve inheritance if the config contains:
+    __inherit__: path/to/parent.yaml or a ListConfig of such paths.
+    """
+    if isinstance(config, DictConfig):
+        inherit = config.pop("__inherit__", None)
+
+        if inherit:
+            inherit_list = inherit if isinstance(inherit, ListConfig) else [inherit]
+
+            parent_config = None
+            for parent_path in inherit_list:
+                assert isinstance(parent_path, str)
+                parent_config = (
+                    load_config(parent_path)
+                    if parent_config is None
+                    else OmegaConf.merge(parent_config, load_config(parent_path))
+                )
+
+            if len(config.keys()) > 0:
+                config = OmegaConf.merge(parent_config, config)
+            else:
+                config = parent_config
+    return config
+
+
+def import_item(path: str, name: str) -> Any:
+    """
+    Import a python item. Example: import_item("path.to.file", "MyClass") -> MyClass
+    """
+    return getattr(importlib.import_module(path), name)
+
+
+def create_object(config: DictConfig) -> Any:
+    """
+    Create an object from config.
+    The config is expected to contains the following:
+    __object__:
+      path: path.to.module
+      name: MyClass
+      args: as_config | as_params (default to as_config)
+    """
+    config = DictConfig(config)
+    item = import_item(
+        path=config.__object__.path,
+        name=config.__object__.name,
+    )
+    args = config.__object__.get("args", "as_config")
+    if args == "as_config":
+        return item(config)
+    if args == "as_params":
+        config = OmegaConf.to_object(config)
+        config.pop("__object__")
+        return item(**config)
+    raise NotImplementedError(f"Unknown args type: {args}")
+
+
+def create_dataset(path: str, *args, **kwargs) -> Any:
+    """
+    Create a dataset. Requires the file to contain a "create_dataset" function.
+    """
+    return import_item(path, "create_dataset")(*args, **kwargs)
+
+
+def to_dict_recursive(config_obj):
+    if isinstance(config_obj, DictConfig):
+        return {k: to_dict_recursive(v) for k, v in config_obj.items()}
+    elif isinstance(config_obj, ListConfig):
+        return [to_dict_recursive(item) for item in config_obj]
+    return config_obj

depth_anything_3/cli.py

@@ -0,0 +1,824 @@
+# flake8: noqa: E402
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Refactored Depth Anything 3 CLI
+Clean, modular command-line interface
+"""
+
+from __future__ import annotations
+
+import os
+import typer
+
+from depth_anything_3.services import start_server
+from depth_anything_3.services.gallery import gallery as gallery_main
+from depth_anything_3.services.inference_service import run_inference
+from depth_anything_3.services.input_handlers import (
+    ColmapHandler,
+    ImageHandler,
+    ImagesHandler,
+    InputHandler,
+    VideoHandler,
+    parse_export_feat,
+)
+from depth_anything_3.utils.constants import (
+    DEFAULT_EXPORT_DIR,
+    DEFAULT_GALLERY_DIR,
+    DEFAULT_GRADIO_DIR,
+    DEFAULT_MODEL,
+)
+
+os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
+
+app = typer.Typer(help="Depth Anything 3 - Video depth estimation CLI", add_completion=False)
+
+
+# ============================================================================
+# Input type detection utilities
+# ============================================================================
+
+# Supported file extensions
+IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tiff", ".tif"}
+VIDEO_EXTENSIONS = {".mp4", ".avi", ".mov", ".mkv", ".flv", ".wmv", ".webm", ".m4v"}
+
+
+def detect_input_type(input_path: str) -> str:
+    """
+    Detect input type from path.
+
+    Returns:
+        - "image": Single image file
+        - "images": Directory containing images
+        - "video": Video file
+        - "colmap": COLMAP directory structure
+        - "unknown": Cannot determine type
+    """
+    if not os.path.exists(input_path):
+        return "unknown"
+
+    # Check if it's a file
+    if os.path.isfile(input_path):
+        ext = os.path.splitext(input_path)[1].lower()
+        if ext in IMAGE_EXTENSIONS:
+            return "image"
+        elif ext in VIDEO_EXTENSIONS:
+            return "video"
+        return "unknown"
+
+    # Check if it's a directory
+    if os.path.isdir(input_path):
+        # Check for COLMAP structure
+        images_dir = os.path.join(input_path, "images")
+        sparse_dir = os.path.join(input_path, "sparse")
+
+        if os.path.isdir(images_dir) and os.path.isdir(sparse_dir):
+            return "colmap"
+
+        # Check if directory contains image files
+        for item in os.listdir(input_path):
+            item_path = os.path.join(input_path, item)
+            if os.path.isfile(item_path):
+                ext = os.path.splitext(item)[1].lower()
+                if ext in IMAGE_EXTENSIONS:
+                    return "images"
+
+        return "unknown"
+
+    return "unknown"
+
+
+# ============================================================================
+# Common parameters and configuration
+# ============================================================================
+
+# ============================================================================
+# Inference commands
+# ============================================================================
+
+
+@app.command()
+def auto(
+    input_path: str = typer.Argument(
+        ..., help="Path to input (image, directory, video, or COLMAP)"
+    ),
+    model_dir: str = typer.Option(DEFAULT_MODEL, help="Model directory path"),
+    export_dir: str = typer.Option(DEFAULT_EXPORT_DIR, help="Export directory"),
+    export_format: str = typer.Option("glb", help="Export format"),
+    device: str = typer.Option("cuda", help="Device to use"),
+    use_backend: bool = typer.Option(False, help="Use backend service for inference"),
+    backend_url: str = typer.Option(
+        "http://localhost:8008", help="Backend URL (default: http://localhost:8008)"
+    ),
+    process_res: int = typer.Option(504, help="Processing resolution"),
+    process_res_method: str = typer.Option(
+        "upper_bound_resize", help="Processing resolution method"
+    ),
+    export_feat: str = typer.Option(
+        "",
+        help="[FEAT_VIS]Export features from specified layers using comma-separated indices (e.g., '0,1,2').",
+    ),
+    auto_cleanup: bool = typer.Option(
+        False, help="Automatically clean export directory if it exists (no prompt)"
+    ),
+    # Video-specific options
+    fps: float = typer.Option(1.0, help="[Video] Sampling FPS for frame extraction"),
+    # COLMAP-specific options
+    sparse_subdir: str = typer.Option(
+        "", help="[COLMAP] Sparse reconstruction subdirectory (e.g., '0' for sparse/0/)"
+    ),
+    align_to_input_ext_scale: bool = typer.Option(
+        True, help="[COLMAP] Align prediction to input extrinsics scale"
+    ),
+    # Pose estimation options
+    use_ray_pose: bool = typer.Option(
+        False, help="Use ray-based pose estimation instead of camera decoder"
+    ),
+    ref_view_strategy: str = typer.Option(
+        "saddle_balanced",
+        help="Reference view selection strategy: empty, first, middle, saddle_balanced, saddle_sim_range",
+    ),
+    # GLB export options
+    conf_thresh_percentile: float = typer.Option(
+        40.0, help="[GLB] Lower percentile for adaptive confidence threshold"
+    ),
+    num_max_points: int = typer.Option(
+        1_000_000, help="[GLB] Maximum number of points in the point cloud"
+    ),
+    show_cameras: bool = typer.Option(
+        True, help="[GLB] Show camera wireframes in the exported scene"
+    ),
+    # Feat_vis export options
+    feat_vis_fps: int = typer.Option(15, help="[FEAT_VIS] Frame rate for output video"),
+):
+    """
+    Automatically detect input type and run appropriate processing.
+
+    Supports:
+    - Single image file (.jpg, .png, etc.)
+    - Directory of images
+    - Video file (.mp4, .avi, etc.)
+    - COLMAP directory (with 'images' and 'sparse' subdirectories)
+    """
+    # Detect input type
+    input_type = detect_input_type(input_path)
+
+    if input_type == "unknown":
+        typer.echo(f"❌ Error: Cannot determine input type for: {input_path}", err=True)
+        typer.echo("Supported inputs:", err=True)
+        typer.echo("  - Single image file (.jpg, .png, etc.)", err=True)
+        typer.echo("  - Directory containing images", err=True)
+        typer.echo("  - Video file (.mp4, .avi, etc.)", err=True)
+        typer.echo("  - COLMAP directory (with 'images/' and 'sparse/' subdirectories)", err=True)
+        raise typer.Exit(1)
+
+    # Display detected type
+    typer.echo(f"🔍 Detected input type: {input_type.upper()}")
+    typer.echo(f"📁 Input path: {input_path}")
+    typer.echo()
+
+    # Determine backend URL based on use_backend flag
+    final_backend_url = backend_url if use_backend else None
+
+    # Parse export_feat parameter
+    export_feat_layers = parse_export_feat(export_feat)
+
+    # Route to appropriate handler
+    if input_type == "image":
+        typer.echo("Processing single image...")
+        # Process input
+        image_files = ImageHandler.process(input_path)
+
+        # Handle export directory
+        export_dir = InputHandler.handle_export_dir(export_dir, auto_cleanup)
+
+        # Run inference
+        run_inference(
+            image_paths=image_files,
+            export_dir=export_dir,
+            model_dir=model_dir,
+            device=device,
+            backend_url=final_backend_url,
+            export_format=export_format,
+            process_res=process_res,
+            process_res_method=process_res_method,
+            export_feat_layers=export_feat_layers,
+            use_ray_pose=use_ray_pose,
+            ref_view_strategy=ref_view_strategy,
+            conf_thresh_percentile=conf_thresh_percentile,
+            num_max_points=num_max_points,
+            show_cameras=show_cameras,
+            feat_vis_fps=feat_vis_fps,
+        )
+
+    elif input_type == "images":
+        typer.echo("Processing directory of images...")
+        # Process input - use default extensions
+        image_files = ImagesHandler.process(input_path, "png,jpg,jpeg")
+
+        # Handle export directory
+        export_dir = InputHandler.handle_export_dir(export_dir, auto_cleanup)
+
+        # Run inference
+        run_inference(
+            image_paths=image_files,
+            export_dir=export_dir,
+            model_dir=model_dir,
+            device=device,
+            backend_url=final_backend_url,
+            export_format=export_format,
+            process_res=process_res,
+            process_res_method=process_res_method,
+            export_feat_layers=export_feat_layers,
+            use_ray_pose=use_ray_pose,
+            ref_view_strategy=ref_view_strategy,
+            conf_thresh_percentile=conf_thresh_percentile,
+            num_max_points=num_max_points,
+            show_cameras=show_cameras,
+            feat_vis_fps=feat_vis_fps,
+        )
+
+    elif input_type == "video":
+        typer.echo(f"Processing video with FPS={fps}...")
+        # Handle export directory
+        export_dir = InputHandler.handle_export_dir(export_dir, auto_cleanup)
+
+        # Process input
+        image_files = VideoHandler.process(input_path, export_dir, fps)
+
+        # Run inference
+        run_inference(
+            image_paths=image_files,
+            export_dir=export_dir,
+            model_dir=model_dir,
+            device=device,
+            backend_url=final_backend_url,
+            export_format=export_format,
+            process_res=process_res,
+            process_res_method=process_res_method,
+            export_feat_layers=export_feat_layers,
+            use_ray_pose=use_ray_pose,
+            ref_view_strategy=ref_view_strategy,
+            conf_thresh_percentile=conf_thresh_percentile,
+            num_max_points=num_max_points,
+            show_cameras=show_cameras,
+            feat_vis_fps=feat_vis_fps,
+        )
+
+    elif input_type == "colmap":
+        typer.echo(
+            f"Processing COLMAP directory (sparse subdirectory: '{sparse_subdir or 'default'}')..."
+        )
+        # Process input
+        image_files, extrinsics, intrinsics = ColmapHandler.process(input_path, sparse_subdir)
+
+        # Handle export directory
+        export_dir = InputHandler.handle_export_dir(export_dir, auto_cleanup)
+
+        # Run inference
+        run_inference(
+            image_paths=image_files,
+            export_dir=export_dir,
+            model_dir=model_dir,
+            device=device,
+            backend_url=final_backend_url,
+            export_format=export_format,
+            process_res=process_res,
+            process_res_method=process_res_method,
+            export_feat_layers=export_feat_layers,
+            extrinsics=extrinsics,
+            intrinsics=intrinsics,
+            align_to_input_ext_scale=align_to_input_ext_scale,
+            use_ray_pose=use_ray_pose,
+            ref_view_strategy=ref_view_strategy,
+            conf_thresh_percentile=conf_thresh_percentile,
+            num_max_points=num_max_points,
+            show_cameras=show_cameras,
+            feat_vis_fps=feat_vis_fps,
+        )
+
+    typer.echo()
+    typer.echo("✅ Processing completed successfully!")
+
+
+@app.command()
+def image(
+    image_path: str = typer.Argument(..., help="Path to input image file"),
+    model_dir: str = typer.Option(DEFAULT_MODEL, help="Model directory path"),
+    export_dir: str = typer.Option(DEFAULT_EXPORT_DIR, help="Export directory"),
+    export_format: str = typer.Option("glb", help="Export format"),
+    device: str = typer.Option("cuda", help="Device to use"),
+    use_backend: bool = typer.Option(False, help="Use backend service for inference"),
+    backend_url: str = typer.Option(
+        "http://localhost:8008", help="Backend URL (default: http://localhost:8008)"
+    ),
+    process_res: int = typer.Option(504, help="Processing resolution"),
+    process_res_method: str = typer.Option(
+        "upper_bound_resize", help="Processing resolution method"
+    ),
+    export_feat: str = typer.Option(
+        "",
+        help="[FEAT_VIS] Export features from specified layers using comma-separated indices (e.g., '0,1,2').",
+    ),
+    auto_cleanup: bool = typer.Option(
+        False, help="Automatically clean export directory if it exists (no prompt)"
+    ),
+    # Pose estimation options
+    use_ray_pose: bool = typer.Option(
+        False, help="Use ray-based pose estimation instead of camera decoder"
+    ),
+    ref_view_strategy: str = typer.Option(
+        "saddle_balanced",
+        help="Reference view selection strategy: empty, first, middle, saddle_balanced, saddle_sim_range",
+    ),
+    # GLB export options
+    conf_thresh_percentile: float = typer.Option(
+        40.0, help="[GLB] Lower percentile for adaptive confidence threshold"
+    ),
+    num_max_points: int = typer.Option(
+        1_000_000, help="[GLB] Maximum number of points in the point cloud"
+    ),
+    show_cameras: bool = typer.Option(
+        True, help="[GLB] Show camera wireframes in the exported scene"
+    ),
+    # Feat_vis export options
+    feat_vis_fps: int = typer.Option(15, help="[FEAT_VIS] Frame rate for output video"),
+):
+    """Run camera pose and depth estimation on a single image."""
+    # Process input
+    image_files = ImageHandler.process(image_path)
+
+    # Handle export directory
+    export_dir = InputHandler.handle_export_dir(export_dir, auto_cleanup)
+
+    # Parse export_feat parameter
+    export_feat_layers = parse_export_feat(export_feat)
+
+    # Determine backend URL based on use_backend flag
+    final_backend_url = backend_url if use_backend else None
+
+    # Run inference
+    run_inference(
+        image_paths=image_files,
+        export_dir=export_dir,
+        model_dir=model_dir,
+        device=device,
+        backend_url=final_backend_url,
+        export_format=export_format,
+        process_res=process_res,
+        process_res_method=process_res_method,
+        export_feat_layers=export_feat_layers,
+        use_ray_pose=use_ray_pose,
+        reference_view_strategy=reference_view_strategy,
+        conf_thresh_percentile=conf_thresh_percentile,
+        num_max_points=num_max_points,
+        show_cameras=show_cameras,
+        feat_vis_fps=feat_vis_fps,
+    )
+
+
+@app.command()
+def images(
+    images_dir: str = typer.Argument(..., help="Path to directory containing input images"),
+    image_extensions: str = typer.Option(
+        "png,jpg,jpeg", help="Comma-separated image file extensions to process"
+    ),
+    model_dir: str = typer.Option(DEFAULT_MODEL, help="Model directory path"),
+    export_dir: str = typer.Option(DEFAULT_EXPORT_DIR, help="Export directory"),
+    export_format: str = typer.Option("glb", help="Export format"),
+    device: str = typer.Option("cuda", help="Device to use"),
+    use_backend: bool = typer.Option(False, help="Use backend service for inference"),
+    backend_url: str = typer.Option(
+        "http://localhost:8008", help="Backend URL (default: http://localhost:8008)"
+    ),
+    process_res: int = typer.Option(504, help="Processing resolution"),
+    process_res_method: str = typer.Option(
+        "upper_bound_resize", help="Processing resolution method"
+    ),
+    export_feat: str = typer.Option(
+        "",
+        help="[FEAT_VIS] Export features from specified layers using comma-separated indices (e.g., '0,1,2').",
+    ),
+    auto_cleanup: bool = typer.Option(
+        False, help="Automatically clean export directory if it exists (no prompt)"
+    ),
+    # Pose estimation options
+    use_ray_pose: bool = typer.Option(
+        False, help="Use ray-based pose estimation instead of camera decoder"
+    ),
+    ref_view_strategy: str = typer.Option(
+        "saddle_balanced",
+        help="Reference view selection strategy: empty, first, middle, saddle_balanced, saddle_sim_range",
+    ),
+    # GLB export options
+    conf_thresh_percentile: float = typer.Option(
+        40.0, help="[GLB] Lower percentile for adaptive confidence threshold"
+    ),
+    num_max_points: int = typer.Option(
+        1_000_000, help="[GLB] Maximum number of points in the point cloud"
+    ),
+    show_cameras: bool = typer.Option(
+        True, help="[GLB] Show camera wireframes in the exported scene"
+    ),
+    # Feat_vis export options
+    feat_vis_fps: int = typer.Option(15, help="[FEAT_VIS] Frame rate for output video"),
+):
+    """Run camera pose and depth estimation on a directory of images."""
+    # Process input
+    image_files = ImagesHandler.process(images_dir, image_extensions)
+
+    # Handle export directory
+    export_dir = InputHandler.handle_export_dir(export_dir, auto_cleanup)
+
+    # Parse export_feat parameter
+    export_feat_layers = parse_export_feat(export_feat)
+
+    # Determine backend URL based on use_backend flag
+    final_backend_url = backend_url if use_backend else None
+
+    # Run inference
+    run_inference(
+        image_paths=image_files,
+        export_dir=export_dir,
+        model_dir=model_dir,
+        device=device,
+        backend_url=final_backend_url,
+        export_format=export_format,
+        process_res=process_res,
+        process_res_method=process_res_method,
+        export_feat_layers=export_feat_layers,
+        use_ray_pose=use_ray_pose,
+        reference_view_strategy=reference_view_strategy,
+        conf_thresh_percentile=conf_thresh_percentile,
+        num_max_points=num_max_points,
+        show_cameras=show_cameras,
+        feat_vis_fps=feat_vis_fps,
+    )
+
+
+@app.command()
+def colmap(
+    colmap_dir: str = typer.Argument(
+        ..., help="Path to COLMAP directory containing 'images' and 'sparse' subdirectories"
+    ),
+    sparse_subdir: str = typer.Option(
+        "", help="Sparse reconstruction subdirectory (e.g., '0' for sparse/0/, empty for sparse/)"
+    ),
+    align_to_input_ext_scale: bool = typer.Option(
+        True, help="Align prediction to input extrinsics scale"
+    ),
+    model_dir: str = typer.Option(DEFAULT_MODEL, help="Model directory path"),
+    export_dir: str = typer.Option(DEFAULT_EXPORT_DIR, help="Export directory"),
+    export_format: str = typer.Option("glb", help="Export format"),
+    device: str = typer.Option("cuda", help="Device to use"),
+    use_backend: bool = typer.Option(False, help="Use backend service for inference"),
+    backend_url: str = typer.Option(
+        "http://localhost:8008", help="Backend URL (default: http://localhost:8008)"
+    ),
+    process_res: int = typer.Option(504, help="Processing resolution"),
+    process_res_method: str = typer.Option(
+        "upper_bound_resize", help="Processing resolution method"
+    ),
+    export_feat: str = typer.Option(
+        "",
+        help="Export features from specified layers using comma-separated indices (e.g., '0,1,2').",
+    ),
+    auto_cleanup: bool = typer.Option(
+        False, help="Automatically clean export directory if it exists (no prompt)"
+    ),
+    # Pose estimation options
+    use_ray_pose: bool = typer.Option(
+        False, help="Use ray-based pose estimation instead of camera decoder"
+    ),
+    ref_view_strategy: str = typer.Option(
+        "saddle_balanced",
+        help="Reference view selection strategy: empty, first, middle, saddle_balanced, saddle_sim_range",
+    ),
+    # GLB export options
+    conf_thresh_percentile: float = typer.Option(
+        40.0, help="[GLB] Lower percentile for adaptive confidence threshold"
+    ),
+    num_max_points: int = typer.Option(
+        1_000_000, help="[GLB] Maximum number of points in the point cloud"
+    ),
+    show_cameras: bool = typer.Option(
+        True, help="[GLB] Show camera wireframes in the exported scene"
+    ),
+    # Feat_vis export options
+    feat_vis_fps: int = typer.Option(15, help="[FEAT_VIS] Frame rate for output video"),
+):
+    """Run pose conditioned depth estimation on COLMAP data."""
+    # Process input
+    image_files, extrinsics, intrinsics = ColmapHandler.process(colmap_dir, sparse_subdir)
+
+    # Handle export directory
+    export_dir = InputHandler.handle_export_dir(export_dir, auto_cleanup)
+
+    # Parse export_feat parameter
+    export_feat_layers = parse_export_feat(export_feat)
+
+    # Determine backend URL based on use_backend flag
+    final_backend_url = backend_url if use_backend else None
+
+    # Run inference
+    run_inference(
+        image_paths=image_files,
+        export_dir=export_dir,
+        model_dir=model_dir,
+        device=device,
+        backend_url=final_backend_url,
+        export_format=export_format,
+        process_res=process_res,
+        process_res_method=process_res_method,
+        export_feat_layers=export_feat_layers,
+        extrinsics=extrinsics,
+        intrinsics=intrinsics,
+        align_to_input_ext_scale=align_to_input_ext_scale,
+        use_ray_pose=use_ray_pose,
+        reference_view_strategy=reference_view_strategy,
+        conf_thresh_percentile=conf_thresh_percentile,
+        num_max_points=num_max_points,
+        show_cameras=show_cameras,
+        feat_vis_fps=feat_vis_fps,
+    )
+
+
+@app.command()
+def video(
+    video_path: str = typer.Argument(..., help="Path to input video file"),
+    fps: float = typer.Option(1.0, help="Sampling FPS for frame extraction"),
+    model_dir: str = typer.Option(DEFAULT_MODEL, help="Model directory path"),
+    export_dir: str = typer.Option(DEFAULT_EXPORT_DIR, help="Export directory"),
+    export_format: str = typer.Option("glb", help="Export format"),
+    device: str = typer.Option("cuda", help="Device to use"),
+    use_backend: bool = typer.Option(False, help="Use backend service for inference"),
+    backend_url: str = typer.Option(
+        "http://localhost:8008", help="Backend URL (default: http://localhost:8008)"
+    ),
+    process_res: int = typer.Option(504, help="Processing resolution"),
+    process_res_method: str = typer.Option(
+        "upper_bound_resize", help="Processing resolution method"
+    ),
+    export_feat: str = typer.Option(
+        "",
+        help="[FEAT_VIS] Export features from specified layers using comma-separated indices (e.g., '0,1,2').",
+    ),
+    auto_cleanup: bool = typer.Option(
+        False, help="Automatically clean export directory if it exists (no prompt)"
+    ),
+    # Pose estimation options
+    use_ray_pose: bool = typer.Option(
+        False, help="Use ray-based pose estimation instead of camera decoder"
+    ),
+    ref_view_strategy: str = typer.Option(
+        "saddle_balanced",
+        help="Reference view selection strategy: empty, first, middle, saddle_balanced, saddle_sim_range",
+    ),
+    # GLB export options
+    conf_thresh_percentile: float = typer.Option(
+        40.0, help="[GLB] Lower percentile for adaptive confidence threshold"
+    ),
+    num_max_points: int = typer.Option(
+        1_000_000, help="[GLB] Maximum number of points in the point cloud"
+    ),
+    show_cameras: bool = typer.Option(
+        True, help="[GLB] Show camera wireframes in the exported scene"
+    ),
+    # Feat_vis export options
+    feat_vis_fps: int = typer.Option(15, help="[FEAT_VIS] Frame rate for output video"),
+):
+    """Run depth estimation on video by extracting frames and processing them."""
+    # Handle export directory
+    export_dir = InputHandler.handle_export_dir(export_dir, auto_cleanup)
+
+    # Process input
+    image_files = VideoHandler.process(video_path, export_dir, fps)
+
+    # Parse export_feat parameter
+    export_feat_layers = parse_export_feat(export_feat)
+
+    # Determine backend URL based on use_backend flag
+    final_backend_url = backend_url if use_backend else None
+
+    # Run inference
+    run_inference(
+        image_paths=image_files,
+        export_dir=export_dir,
+        model_dir=model_dir,
+        device=device,
+        backend_url=final_backend_url,
+        export_format=export_format,
+        process_res=process_res,
+        process_res_method=process_res_method,
+        export_feat_layers=export_feat_layers,
+        use_ray_pose=use_ray_pose,
+        reference_view_strategy=reference_view_strategy,
+        conf_thresh_percentile=conf_thresh_percentile,
+        num_max_points=num_max_points,
+        show_cameras=show_cameras,
+        feat_vis_fps=feat_vis_fps,
+    )
+
+
+# ============================================================================
+# Service management commands
+# ============================================================================
+
+
+@app.command()
+def backend(
+    model_dir: str = typer.Option(DEFAULT_MODEL, help="Model directory path"),
+    device: str = typer.Option("cuda", help="Device to use"),
+    host: str = typer.Option("127.0.0.1", help="Host to bind to"),
+    port: int = typer.Option(8008, help="Port to bind to"),
+    gallery_dir: str = typer.Option(DEFAULT_GALLERY_DIR, help="Gallery directory path (optional)"),
+    api_key: str = typer.Option(
+        None,
+        help="Require this API key (X-API-Key header) on inference requests. Falls back "
+        "to the DA3_BACKEND_API_KEY env var, or an auto-generated key when binding to a "
+        "non-loopback host.",
+    ),
+    allow_unauthenticated: bool = typer.Option(
+        False,
+        help="Skip authentication entirely on a non-loopback host, instead of using an "
+        "auto-generated API key.",
+    ),
+):
+    """Start model backend service with integrated gallery."""
+    typer.echo("=" * 60)
+    typer.echo("🚀 Starting Depth Anything 3 Backend Server")
+    typer.echo("=" * 60)
+    typer.echo(f"Model directory: {model_dir}")
+    typer.echo(f"Device: {device}")
+
+    # The gallery directory is also where /inference writes exports, so make sure it
+    # exists up front rather than treating a fresh checkout as "no gallery configured".
+    if gallery_dir:
+        os.makedirs(gallery_dir, exist_ok=True)
+        typer.echo(f"Gallery directory: {gallery_dir}")
+    else:
+        gallery_dir = None
+
+    typer.echo()
+    typer.echo("📡 Server URLs (Ctrl/CMD+Click to open):")
+    typer.echo(f"  🏠 Home:      http://{host}:{port}")
+    typer.echo(f"  📊 Dashboard: http://{host}:{port}/dashboard")
+    typer.echo(f"  📈 API Status: http://{host}:{port}/status")
+
+    if gallery_dir:
+        typer.echo(f"  🎨 Gallery:   http://{host}:{port}/gallery/")
+
+    typer.echo("=" * 60)
+
+    try:
+        start_server(
+            model_dir,
+            device,
+            host,
+            port,
+            gallery_dir,
+            api_key=api_key,
+            allow_unauthenticated=allow_unauthenticated,
+        )
+    except KeyboardInterrupt:
+        typer.echo("\n👋 Backend server stopped.")
+    except Exception as e:
+        typer.echo(f"❌ Failed to start backend: {e}")
+        raise typer.Exit(1)
+
+
+# ============================================================================
+# Application launch commands
+# ============================================================================
+
+
+@app.command()
+def gradio(
+    model_dir: str = typer.Option(DEFAULT_MODEL, help="Model directory path"),
+    workspace_dir: str = typer.Option(DEFAULT_GRADIO_DIR, help="Workspace directory path"),
+    gallery_dir: str = typer.Option(DEFAULT_GALLERY_DIR, help="Gallery directory path"),
+    host: str = typer.Option("127.0.0.1", help="Host address to bind to"),
+    port: int = typer.Option(7860, help="Port number to bind to"),
+    share: bool = typer.Option(False, help="Create a public link for the app"),
+    debug: bool = typer.Option(False, help="Enable debug mode"),
+    cache_examples: bool = typer.Option(
+        False, help="Pre-cache all example scenes at startup for faster loading"
+    ),
+    cache_gs_tag: str = typer.Option(
+        "",
+        help="Tag to match scene names for high-res+3DGS caching (e.g., 'dl3dv'). Scenes containing this tag will use high_res and infer_gs=True; others will use low_res only.",
+    ),
+):
+    """Launch Depth Anything 3 Gradio interactive web application"""
+    from depth_anything_3.app.gradio_app import DepthAnything3App
+
+    # Create necessary directories
+    os.makedirs(workspace_dir, exist_ok=True)
+    os.makedirs(gallery_dir, exist_ok=True)
+
+    typer.echo("Launching Depth Anything 3 Gradio application...")
+    typer.echo(f"Model directory: {model_dir}")
+    typer.echo(f"Workspace directory: {workspace_dir}")
+    typer.echo(f"Gallery directory: {gallery_dir}")
+    typer.echo(f"Host: {host}")
+    typer.echo(f"Port: {port}")
+    typer.echo(f"Share: {share}")
+    typer.echo(f"Debug mode: {debug}")
+    typer.echo(f"Cache examples: {cache_examples}")
+    if cache_examples:
+        if cache_gs_tag:
+            typer.echo(
+                f"Cache GS Tag: '{cache_gs_tag}' (scenes matching this tag will use high-res + 3DGS)"
+            )
+        else:
+            typer.echo(f"Cache GS Tag: None (all scenes will use low-res only)")
+
+    try:
+        # Initialize and launch application
+        app = DepthAnything3App(
+            model_dir=model_dir, workspace_dir=workspace_dir, gallery_dir=gallery_dir
+        )
+
+        # Pre-cache examples if requested
+        if cache_examples:
+            typer.echo("\n" + "=" * 60)
+            typer.echo("Pre-caching mode enabled")
+            if cache_gs_tag:
+                typer.echo(f"Scenes containing '{cache_gs_tag}' will use HIGH-RES + 3DGS")
+                typer.echo(f"Other scenes will use LOW-RES only")
+            else:
+                typer.echo(f"All scenes will use LOW-RES only")
+            typer.echo("=" * 60)
+            app.cache_examples(
+                show_cam=True,
+                filter_black_bg=False,
+                filter_white_bg=False,
+                save_percentage=20.0,
+                num_max_points=1000,
+                cache_gs_tag=cache_gs_tag,
+                gs_trj_mode="smooth",
+                gs_video_quality="low",
+            )
+
+        # Prepare launch arguments
+        launch_kwargs = {"share": share, "debug": debug}
+
+        app.launch(host=host, port=port, **launch_kwargs)
+
+    except KeyboardInterrupt:
+        typer.echo("\nGradio application stopped.")
+    except Exception as e:
+        typer.echo(f"Failed to launch Gradio application: {e}")
+        raise typer.Exit(1)
+
+
+@app.command()
+def gallery(
+    gallery_dir: str = typer.Option(DEFAULT_GALLERY_DIR, help="Gallery root directory"),
+    host: str = typer.Option("127.0.0.1", help="Host address to bind to"),
+    port: int = typer.Option(8007, help="Port number to bind to"),
+    open_browser: bool = typer.Option(False, help="Open browser after launch"),
+):
+    """Launch Depth Anything 3 Gallery server"""
+
+    # Validate gallery directory
+    if not os.path.exists(gallery_dir):
+        raise typer.BadParameter(f"Gallery directory not found: {gallery_dir}")
+
+    typer.echo("Launching Depth Anything 3 Gallery server...")
+    typer.echo(f"Gallery directory: {gallery_dir}")
+    typer.echo(f"Host: {host}")
+    typer.echo(f"Port: {port}")
+    typer.echo(f"Auto-open browser: {open_browser}")
+
+    try:
+        # Set command line arguments
+        import sys
+
+        sys.argv = ["gallery", "--dir", gallery_dir, "--host", host, "--port", str(port)]
+        if open_browser:
+            sys.argv.append("--open")
+
+        # Launch gallery server
+        gallery_main()
+
+    except KeyboardInterrupt:
+        typer.echo("\nGallery server stopped.")
+    except Exception as e:
+        typer.echo(f"Failed to launch Gallery server: {e}")
+        raise typer.Exit(1)
+
+
+if __name__ == "__main__":
+    app()

depth_anything_3/configs/da3-base.yaml

@@ -0,0 +1,45 @@
+__object__:
+  path: depth_anything_3.model.da3
+  name: DepthAnything3Net
+  args: as_params
+
+net:
+  __object__:
+    path: depth_anything_3.model.dinov2.dinov2
+    name: DinoV2
+    args: as_params
+
+  name: vitb
+  out_layers: [5, 7, 9, 11]
+  alt_start: 4
+  qknorm_start: 4
+  rope_start: 4
+  cat_token: True
+
+head:
+  __object__:
+    path: depth_anything_3.model.dualdpt
+    name: DualDPT
+    args: as_params
+
+  dim_in: &head_dim_in 1536
+  output_dim: 2
+  features: &head_features 128
+  out_channels: &head_out_channels [96, 192, 384, 768]
+
+
+cam_enc:
+  __object__:
+    path: depth_anything_3.model.cam_enc
+    name: CameraEnc
+    args: as_params
+
+  dim_out: 768
+
+cam_dec:
+  __object__:
+    path: depth_anything_3.model.cam_dec
+    name: CameraDec
+    args: as_params
+
+  dim_in: 1536

depth_anything_3/configs/da3-giant.yaml

@@ -0,0 +1,71 @@
+__object__:
+  path: depth_anything_3.model.da3
+  name: DepthAnything3Net
+  args: as_params
+
+net:
+  __object__:
+    path: depth_anything_3.model.dinov2.dinov2
+    name: DinoV2
+    args: as_params
+
+  name: vitg
+  out_layers: [19, 27, 33, 39]
+  alt_start: 13
+  qknorm_start: 13
+  rope_start: 13
+  cat_token: True
+
+head:
+  __object__:
+    path: depth_anything_3.model.dualdpt
+    name: DualDPT
+    args: as_params
+
+  dim_in: &head_dim_in 3072
+  output_dim: 2
+  features: &head_features 256
+  out_channels: &head_out_channels [256, 512, 1024, 1024]
+
+
+cam_enc:
+  __object__:
+    path: depth_anything_3.model.cam_enc
+    name: CameraEnc
+    args: as_params
+
+  dim_out: 1536
+
+cam_dec:
+  __object__:
+    path: depth_anything_3.model.cam_dec
+    name: CameraDec
+    args: as_params
+
+  dim_in: 3072
+
+
+gs_head:
+  __object__:
+    path: depth_anything_3.model.gsdpt
+    name: GSDPT
+    args: as_params
+
+  dim_in: *head_dim_in
+  output_dim: 38  # should align with gs_adapter's setting, for gs params
+  features: *head_features
+  out_channels: *head_out_channels
+
+
+gs_adapter:
+  __object__:
+    path: depth_anything_3.model.gs_adapter
+    name: GaussianAdapter
+    args: as_params
+
+  sh_degree: 2
+  pred_color: false  # predict SH coefficient if false
+  pred_offset_depth: true
+  pred_offset_xy: true
+  gaussian_scale_min: 1e-5
+  gaussian_scale_max: 30.0

depth_anything_3/configs/da3-large.yaml

@@ -0,0 +1,45 @@
+__object__:
+  path: depth_anything_3.model.da3
+  name: DepthAnything3Net
+  args: as_params
+
+net:
+  __object__:
+    path: depth_anything_3.model.dinov2.dinov2
+    name: DinoV2
+    args: as_params
+
+  name: vitl
+  out_layers: [11, 15, 19, 23]
+  alt_start: 8
+  qknorm_start: 8
+  rope_start: 8
+  cat_token: True
+
+head:
+  __object__:
+    path: depth_anything_3.model.dualdpt
+    name: DualDPT
+    args: as_params
+
+  dim_in: &head_dim_in 2048
+  output_dim: 2
+  features: &head_features 256
+  out_channels: &head_out_channels [256, 512, 1024, 1024]
+
+
+cam_enc:
+  __object__:
+    path: depth_anything_3.model.cam_enc
+    name: CameraEnc
+    args: as_params
+
+  dim_out: 1024
+
+cam_dec:
+  __object__:
+    path: depth_anything_3.model.cam_dec
+    name: CameraDec
+    args: as_params
+
+  dim_in: 2048

depth_anything_3/configs/da3-small.yaml

@@ -0,0 +1,45 @@
+__object__:
+  path: depth_anything_3.model.da3
+  name: DepthAnything3Net
+  args: as_params
+
+net:
+  __object__:
+    path: depth_anything_3.model.dinov2.dinov2
+    name: DinoV2
+    args: as_params
+
+  name: vits
+  out_layers: [5, 7, 9, 11]
+  alt_start: 4
+  qknorm_start: 4
+  rope_start: 4
+  cat_token: True
+
+head:
+  __object__:
+    path: depth_anything_3.model.dualdpt
+    name: DualDPT
+    args: as_params
+
+  dim_in: &head_dim_in 768
+  output_dim: 2
+  features: &head_features 64
+  out_channels: &head_out_channels [48, 96, 192, 384]
+
+
+cam_enc:
+  __object__:
+    path: depth_anything_3.model.cam_enc
+    name: CameraEnc
+    args: as_params
+
+  dim_out: 384
+
+cam_dec:
+  __object__:
+    path: depth_anything_3.model.cam_dec
+    name: CameraDec
+    args: as_params
+
+  dim_in: 768

depth_anything_3/configs/da3metric-large.yaml

@@ -0,0 +1,28 @@
+__object__:
+  path: depth_anything_3.model.da3
+  name: DepthAnything3Net
+  args: as_params
+
+net:
+  __object__:
+    path: depth_anything_3.model.dinov2.dinov2
+    name: DinoV2
+    args: as_params
+
+  name: vitl
+  out_layers: [4, 11, 17, 23]
+  alt_start: -1 # -1 means disable
+  qknorm_start: -1
+  rope_start: -1
+  cat_token: False
+
+head:
+  __object__:
+    path: depth_anything_3.model.dpt
+    name: DPT
+    args: as_params
+
+  dim_in: 1024
+  output_dim: 1
+  features: 256
+  out_channels: [256, 512, 1024, 1024]

depth_anything_3/configs/da3mono-large.yaml

@@ -0,0 +1,28 @@
+__object__:
+  path: depth_anything_3.model.da3
+  name: DepthAnything3Net
+  args: as_params
+
+net:
+  __object__:
+    path: depth_anything_3.model.dinov2.dinov2
+    name: DinoV2
+    args: as_params
+
+  name: vitl
+  out_layers: [4, 11, 17, 23]
+  alt_start: -1 # -1 means disable
+  qknorm_start: -1
+  rope_start: -1
+  cat_token: False
+
+head:
+  __object__:
+    path: depth_anything_3.model.dpt
+    name: DPT
+    args: as_params
+
+  dim_in: 1024
+  output_dim: 1
+  features: 256
+  out_channels: [256, 512, 1024, 1024]

depth_anything_3/configs/da3nested-giant-large.yaml

@@ -0,0 +1,10 @@
+__object__:
+  path: depth_anything_3.model.da3
+  name: NestedDepthAnything3Net
+  args: as_params
+
+anyview:
+  __inherit__: depth_anything_3.configs.da3-giant
+
+metric:
+  __inherit__: depth_anything_3.configs.da3metric-large

depth_anything_3/model/__init__.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from depth_anything_3.model.da3 import DepthAnything3Net, NestedDepthAnything3Net
+
+__export__ = [
+    NestedDepthAnything3Net,
+    DepthAnything3Net,
+]

depth_anything_3/model/cam_dec.py

@@ -0,0 +1,45 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+import torch.nn as nn
+
+
+class CameraDec(nn.Module):
+    def __init__(self, dim_in=1536):
+        super().__init__()
+        output_dim = dim_in
+        self.backbone = nn.Sequential(
+            nn.Linear(output_dim, output_dim),
+            nn.ReLU(),
+            nn.Linear(output_dim, output_dim),
+            nn.ReLU(),
+        )
+        self.fc_t = nn.Linear(output_dim, 3)
+        self.fc_qvec = nn.Linear(output_dim, 4)
+        self.fc_fov = nn.Sequential(nn.Linear(output_dim, 2), nn.ReLU())
+
+    def forward(self, feat, camera_encoding=None, *args, **kwargs):
+        B, N = feat.shape[:2]
+        feat = feat.reshape(B * N, -1)
+        feat = self.backbone(feat)
+        out_t = self.fc_t(feat.float()).reshape(B, N, 3)
+        if camera_encoding is None:
+            out_qvec = self.fc_qvec(feat.float()).reshape(B, N, 4)
+            out_fov = self.fc_fov(feat.float()).reshape(B, N, 2)
+        else:
+            out_qvec = camera_encoding[..., 3:7]
+            out_fov = camera_encoding[..., -2:]
+        pose_enc = torch.cat([out_t, out_qvec, out_fov], dim=-1)
+        return pose_enc

depth_anything_3/model/cam_enc.py

@@ -0,0 +1,80 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch.nn as nn
+
+from depth_anything_3.model.utils.attention import Mlp
+from depth_anything_3.model.utils.block import Block
+from depth_anything_3.model.utils.transform import extri_intri_to_pose_encoding
+from depth_anything_3.utils.geometry import affine_inverse
+
+
+class CameraEnc(nn.Module):
+    """
+    CameraHead predicts camera parameters from token representations using iterative refinement.
+
+    It applies a series of transformer blocks (the "trunk") to dedicated camera tokens.
+    """
+
+    def __init__(
+        self,
+        dim_out: int = 1024,
+        dim_in: int = 9,
+        trunk_depth: int = 4,
+        target_dim: int = 9,
+        num_heads: int = 16,
+        mlp_ratio: int = 4,
+        init_values: float = 0.01,
+        **kwargs,
+    ):
+        super().__init__()
+        self.target_dim = target_dim
+        self.trunk_depth = trunk_depth
+        self.trunk = nn.Sequential(
+            *[
+                Block(
+                    dim=dim_out,
+                    num_heads=num_heads,
+                    mlp_ratio=mlp_ratio,
+                    init_values=init_values,
+                )
+                for _ in range(trunk_depth)
+            ]
+        )
+        self.token_norm = nn.LayerNorm(dim_out)
+        self.trunk_norm = nn.LayerNorm(dim_out)
+        self.pose_branch = Mlp(
+            in_features=dim_in,
+            hidden_features=dim_out // 2,
+            out_features=dim_out,
+            drop=0,
+        )
+
+    def forward(
+        self,
+        ext,
+        ixt,
+        image_size,
+    ) -> tuple:
+        c2ws = affine_inverse(ext)
+        pose_encoding = extri_intri_to_pose_encoding(
+            c2ws,
+            ixt,
+            image_size,
+        )
+        pose_tokens = self.pose_branch(pose_encoding)
+        pose_tokens = self.token_norm(pose_tokens)
+        pose_tokens = self.trunk(pose_tokens)
+        pose_tokens = self.trunk_norm(pose_tokens)
+        return pose_tokens

depth_anything_3/model/da3.py

@@ -0,0 +1,442 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import torch
+import torch.nn as nn
+from addict import Dict
+from omegaconf import DictConfig, OmegaConf
+
+from depth_anything_3.cfg import create_object
+from depth_anything_3.model.utils.transform import pose_encoding_to_extri_intri
+from depth_anything_3.utils.alignment import (
+    apply_metric_scaling,
+    compute_alignment_mask,
+    compute_sky_mask,
+    least_squares_scale_scalar,
+    sample_tensor_for_quantile,
+    set_sky_regions_to_max_depth,
+)
+from depth_anything_3.utils.geometry import affine_inverse, as_homogeneous, map_pdf_to_opacity
+from depth_anything_3.utils.ray_utils import get_extrinsic_from_camray
+
+
+def _wrap_cfg(cfg_obj):
+    return OmegaConf.create(cfg_obj)
+
+
+class DepthAnything3Net(nn.Module):
+    """
+    Depth Anything 3 network for depth estimation and camera pose estimation.
+
+    This network consists of:
+    - Backbone: DinoV2 feature extractor
+    - Head: DPT or DualDPT for depth prediction
+    - Optional camera decoders for pose estimation
+    - Optional GSDPT for 3DGS prediction
+
+    Args:
+        preset: Configuration preset containing network dimensions and settings
+
+    Returns:
+        Dictionary containing:
+        - depth: Predicted depth map (B, H, W)
+        - depth_conf: Depth confidence map (B, H, W)
+        - extrinsics: Camera extrinsics (B, N, 4, 4)
+        - intrinsics: Camera intrinsics (B, N, 3, 3)
+        - gaussians: 3D Gaussian Splats (world space), type: model.gs_adapter.Gaussians
+        - aux: Auxiliary features for specified layers
+    """
+
+    # Patch size for feature extraction
+    PATCH_SIZE = 14
+
+    def __init__(self, net, head, cam_dec=None, cam_enc=None, gs_head=None, gs_adapter=None):
+        """
+        Initialize DepthAnything3Net with given yaml-initialized configuration.
+        """
+        super().__init__()
+        self.backbone = net if isinstance(net, nn.Module) else create_object(_wrap_cfg(net))
+        self.head = head if isinstance(head, nn.Module) else create_object(_wrap_cfg(head))
+        self.cam_dec, self.cam_enc = None, None
+        if cam_dec is not None:
+            self.cam_dec = (
+                cam_dec if isinstance(cam_dec, nn.Module) else create_object(_wrap_cfg(cam_dec))
+            )
+            self.cam_enc = (
+                cam_enc if isinstance(cam_enc, nn.Module) else create_object(_wrap_cfg(cam_enc))
+            )
+        self.gs_adapter, self.gs_head = None, None
+        if gs_head is not None and gs_adapter is not None:
+            self.gs_adapter = (
+                gs_adapter
+                if isinstance(gs_adapter, nn.Module)
+                else create_object(_wrap_cfg(gs_adapter))
+            )
+            gs_out_dim = self.gs_adapter.d_in + 1
+            if isinstance(gs_head, nn.Module):
+                assert (
+                    gs_head.out_dim == gs_out_dim
+                ), f"gs_head.out_dim should be {gs_out_dim}, got {gs_head.out_dim}"
+                self.gs_head = gs_head
+            else:
+                assert (
+                    gs_head["output_dim"] == gs_out_dim
+                ), f"gs_head output_dim should set to {gs_out_dim}, got {gs_head['output_dim']}"
+                self.gs_head = create_object(_wrap_cfg(gs_head))
+
+    def forward(
+        self,
+        x: torch.Tensor,
+        extrinsics: torch.Tensor | None = None,
+        intrinsics: torch.Tensor | None = None,
+        export_feat_layers: list[int] | None = [],
+        infer_gs: bool = False,
+        use_ray_pose: bool = False,
+        ref_view_strategy: str = "saddle_balanced",
+    ) -> Dict[str, torch.Tensor]:
+        """
+        Forward pass through the network.
+
+        Args:
+            x: Input images (B, N, 3, H, W)
+            extrinsics: Camera extrinsics (B, N, 4, 4) 
+            intrinsics: Camera intrinsics (B, N, 3, 3) 
+            feat_layers: List of layer indices to extract features from
+            infer_gs: Enable Gaussian Splatting branch
+            use_ray_pose: Use ray-based pose estimation
+            ref_view_strategy: Strategy for selecting reference view
+
+        Returns:
+            Dictionary containing predictions and auxiliary features
+        """
+        # Extract features using backbone
+        if extrinsics is not None:
+            with torch.autocast(device_type=x.device.type, enabled=False):
+                cam_token = self.cam_enc(extrinsics, intrinsics, x.shape[-2:])
+        else:
+            cam_token = None
+
+        feats, aux_feats = self.backbone(
+            x, cam_token=cam_token, export_feat_layers=export_feat_layers, ref_view_strategy=ref_view_strategy
+        )
+        # feats = [[item for item in feat] for feat in feats]
+        H, W = x.shape[-2], x.shape[-1]
+
+        # Process features through depth head
+        with torch.autocast(device_type=x.device.type, enabled=False):
+            output = self._process_depth_head(feats, H, W)
+            if use_ray_pose:
+                output = self._process_ray_pose_estimation(output, H, W)
+            else:
+                output = self._process_camera_estimation(feats, H, W, output)
+            if infer_gs:
+                output = self._process_gs_head(feats, H, W, output, x, extrinsics, intrinsics)
+        
+        output = self._process_mono_sky_estimation(output)    
+
+        # Extract auxiliary features if requested
+        output.aux = self._extract_auxiliary_features(aux_feats, export_feat_layers, H, W)
+
+        return output
+
+    def _process_mono_sky_estimation(
+        self, output: Dict[str, torch.Tensor]
+    ) -> Dict[str, torch.Tensor]:
+        """Process mono sky estimation."""
+        if "sky" not in output:
+            return output
+        non_sky_mask = compute_sky_mask(output.sky, threshold=0.3)
+        if non_sky_mask.sum() <= 10:
+            return output
+        if (~non_sky_mask).sum() <= 10:
+            return output
+        
+        non_sky_depth = output.depth[non_sky_mask]
+        if non_sky_depth.numel() > 100000:
+            idx = torch.randint(0, non_sky_depth.numel(), (100000,), device=non_sky_depth.device)
+            sampled_depth = non_sky_depth[idx]
+        else:
+            sampled_depth = non_sky_depth
+        non_sky_max = torch.quantile(sampled_depth, 0.99)
+
+        # Set sky regions to maximum depth and high confidence
+        output.depth, _ = set_sky_regions_to_max_depth(
+            output.depth, None, non_sky_mask, max_depth=non_sky_max
+        )
+        return output
+
+    def _process_ray_pose_estimation(
+        self, output: Dict[str, torch.Tensor], height: int, width: int
+    ) -> Dict[str, torch.Tensor]:
+        """Process ray pose estimation if ray pose decoder is available."""
+        if "ray" in output and "ray_conf" in output:
+            pred_extrinsic, pred_focal_lengths, pred_principal_points = get_extrinsic_from_camray(
+                output.ray,
+                output.ray_conf,
+                output.ray.shape[-3],
+                output.ray.shape[-2],
+            )
+            pred_extrinsic = affine_inverse(pred_extrinsic) # w2c -> c2w
+            pred_extrinsic = pred_extrinsic[:, :, :3, :]
+            pred_intrinsic = torch.eye(3, 3)[None, None].repeat(pred_extrinsic.shape[0], pred_extrinsic.shape[1], 1, 1).clone().to(pred_extrinsic.device)
+            pred_intrinsic[:, :, 0, 0] = pred_focal_lengths[:, :, 0] / 2 * width
+            pred_intrinsic[:, :, 1, 1] = pred_focal_lengths[:, :, 1] / 2 * height
+            pred_intrinsic[:, :, 0, 2] = pred_principal_points[:, :, 0] * width * 0.5
+            pred_intrinsic[:, :, 1, 2] = pred_principal_points[:, :, 1] * height * 0.5
+            del output.ray
+            del output.ray_conf
+            output.extrinsics = pred_extrinsic
+            output.intrinsics = pred_intrinsic
+        return output
+
+    def _process_depth_head(
+        self, feats: list[torch.Tensor], H: int, W: int
+    ) -> Dict[str, torch.Tensor]:
+        """Process features through the depth prediction head."""
+        return self.head(feats, H, W, patch_start_idx=0)
+
+    def _process_camera_estimation(
+        self, feats: list[torch.Tensor], H: int, W: int, output: Dict[str, torch.Tensor]
+    ) -> Dict[str, torch.Tensor]:
+        """Process camera pose estimation if camera decoder is available."""
+        if self.cam_dec is not None:
+            pose_enc = self.cam_dec(feats[-1][1])
+            # Remove ray information as it's not needed for pose estimation
+            if "ray" in output:
+                del output.ray
+            if "ray_conf" in output:
+                del output.ray_conf
+
+            # Convert pose encoding to extrinsics and intrinsics
+            c2w, ixt = pose_encoding_to_extri_intri(pose_enc, (H, W))
+            output.extrinsics = affine_inverse(c2w)
+            output.intrinsics = ixt
+
+        return output
+
+    def _process_gs_head(
+        self,
+        feats: list[torch.Tensor],
+        H: int,
+        W: int,
+        output: Dict[str, torch.Tensor],
+        in_images: torch.Tensor,
+        extrinsics: torch.Tensor | None = None,
+        intrinsics: torch.Tensor | None = None,
+    ) -> Dict[str, torch.Tensor]:
+        """Process 3DGS parameters estimation if 3DGS head is available."""
+        if self.gs_head is None or self.gs_adapter is None:
+            return output
+        assert output.get("depth", None) is not None, "must provide MV depth for the GS head."
+
+        # The depth is defined in the DA3 model's camera space,
+        # so even with provided GT camera poses,
+        # we instead use the predicted camera poses for better alignment.
+        ctx_extr = output.get("extrinsics", None)
+        ctx_intr = output.get("intrinsics", None)
+        assert (
+            ctx_extr is not None and ctx_intr is not None
+        ), "must process camera info first if GT is not available"
+
+        gt_extr = extrinsics
+        # homo the extr if needed
+        ctx_extr = as_homogeneous(ctx_extr)
+        if gt_extr is not None:
+            gt_extr = as_homogeneous(gt_extr)
+
+        # forward through the gs_dpt head to get 'camera space' parameters
+        gs_outs = self.gs_head(
+            feats=feats,
+            H=H,
+            W=W,
+            patch_start_idx=0,
+            images=in_images,
+        )
+        raw_gaussians = gs_outs.raw_gs
+        densities = gs_outs.raw_gs_conf
+
+        # convert to 'world space' 3DGS parameters; ready to export and render
+        # gt_extr could be None, and will be used to align the pose scale if available
+        gs_world = self.gs_adapter(
+            extrinsics=ctx_extr,
+            intrinsics=ctx_intr,
+            depths=output.depth,
+            opacities=map_pdf_to_opacity(densities),
+            raw_gaussians=raw_gaussians,
+            image_shape=(H, W),
+            gt_extrinsics=gt_extr,
+        )
+        output.gaussians = gs_world
+
+        return output
+
+    def _extract_auxiliary_features(
+        self, feats: list[torch.Tensor], feat_layers: list[int], H: int, W: int
+    ) -> Dict[str, torch.Tensor]:
+        """Extract auxiliary features from specified layers."""
+        aux_features = Dict()
+        assert len(feats) == len(feat_layers)
+        for feat, feat_layer in zip(feats, feat_layers):
+            # Reshape features to spatial dimensions
+            feat_reshaped = feat.reshape(
+                [
+                    feat.shape[0],
+                    feat.shape[1],
+                    H // self.PATCH_SIZE,
+                    W // self.PATCH_SIZE,
+                    feat.shape[-1],
+                ]
+            )
+            aux_features[f"feat_layer_{feat_layer}"] = feat_reshaped
+
+        return aux_features
+
+
+class NestedDepthAnything3Net(nn.Module):
+    """
+    Nested Depth Anything 3 network with metric scaling capabilities.
+
+    This network combines two DepthAnything3Net branches:
+    - Main branch: Standard depth estimation
+    - Metric branch: Metric depth estimation for scaling alignment
+
+    The network performs depth alignment using least squares scaling
+    and handles sky region masking for improved depth estimation.
+
+    Args:
+        preset: Configuration for the main depth estimation branch
+        second_preset: Configuration for the metric depth branch
+    """
+
+    def __init__(self, anyview: DictConfig, metric: DictConfig):
+        """
+        Initialize NestedDepthAnything3Net with two branches.
+
+        Args:
+            preset: Configuration for main depth estimation branch
+            second_preset: Configuration for metric depth branch
+        """
+        super().__init__()
+        self.da3 = create_object(anyview)
+        self.da3_metric = create_object(metric)
+
+    def forward(
+        self,
+        x: torch.Tensor,
+        extrinsics: torch.Tensor | None = None,
+        intrinsics: torch.Tensor | None = None,
+        export_feat_layers: list[int] | None = [],
+        infer_gs: bool = False,
+        use_ray_pose: bool = False,
+        ref_view_strategy: str = "saddle_balanced",
+    ) -> Dict[str, torch.Tensor]:
+        """
+        Forward pass through both branches with metric scaling alignment.
+
+        Args:
+            x: Input images (B, N, 3, H, W)
+            extrinsics: Camera extrinsics (B, N, 4, 4) - unused
+            intrinsics: Camera intrinsics (B, N, 3, 3) - unused
+            feat_layers: List of layer indices to extract features from
+            infer_gs: Enable Gaussian Splatting branch
+            use_ray_pose: Use ray-based pose estimation
+            ref_view_strategy: Strategy for selecting reference view
+
+        Returns:
+            Dictionary containing aligned depth predictions and camera parameters
+        """
+        # Get predictions from both branches
+        output = self.da3(
+            x, extrinsics, intrinsics, export_feat_layers=export_feat_layers, infer_gs=infer_gs, use_ray_pose=use_ray_pose, ref_view_strategy=ref_view_strategy
+        )
+        metric_output = self.da3_metric(x)
+
+        # Apply metric scaling and alignment
+        output = self._apply_metric_scaling(output, metric_output)
+        output = self._apply_depth_alignment(output, metric_output)
+        output = self._handle_sky_regions(output, metric_output)
+
+        return output
+
+    def _apply_metric_scaling(
+        self, output: Dict[str, torch.Tensor], metric_output: Dict[str, torch.Tensor]
+    ) -> Dict[str, torch.Tensor]:
+        """Apply metric scaling to the metric depth output."""
+        # Scale metric depth based on camera intrinsics
+        metric_output.depth = apply_metric_scaling(
+            metric_output.depth,
+            output.intrinsics,
+        )
+        return output
+
+    def _apply_depth_alignment(
+        self, output: Dict[str, torch.Tensor], metric_output: Dict[str, torch.Tensor]
+    ) -> Dict[str, torch.Tensor]:
+        """Apply depth alignment using least squares scaling."""
+        # Compute non-sky mask
+        non_sky_mask = compute_sky_mask(metric_output.sky, threshold=0.3)
+
+        # Ensure we have enough non-sky pixels
+        assert non_sky_mask.sum() > 10, "Insufficient non-sky pixels for alignment"
+
+        # Sample depth confidence for quantile computation
+        depth_conf_ns = output.depth_conf[non_sky_mask]
+        depth_conf_sampled = sample_tensor_for_quantile(depth_conf_ns, max_samples=100000)
+        median_conf = torch.quantile(depth_conf_sampled, 0.5)
+
+        # Compute alignment mask
+        align_mask = compute_alignment_mask(
+            output.depth_conf, non_sky_mask, output.depth, metric_output.depth, median_conf
+        )
+
+        # Compute scale factor using least squares
+        valid_depth = output.depth[align_mask]
+        valid_metric_depth = metric_output.depth[align_mask]
+        scale_factor = least_squares_scale_scalar(valid_metric_depth, valid_depth)
+
+        # Apply scaling to depth and extrinsics
+        output.depth *= scale_factor
+        output.extrinsics[:, :, :3, 3] *= scale_factor
+        output.is_metric = 1
+        output.scale_factor = scale_factor.item()
+
+        return output
+
+    def _handle_sky_regions(
+        self,
+        output: Dict[str, torch.Tensor],
+        metric_output: Dict[str, torch.Tensor],
+        sky_depth_def: float = 200.0,
+    ) -> Dict[str, torch.Tensor]:
+        """Handle sky regions by setting them to maximum depth."""
+        non_sky_mask = compute_sky_mask(metric_output.sky, threshold=0.3)
+
+        # Compute maximum depth for non-sky regions
+        # Use sampling to safely compute quantile on large tensors
+        non_sky_depth = output.depth[non_sky_mask]
+        if non_sky_depth.numel() > 100000:
+            idx = torch.randint(0, non_sky_depth.numel(), (100000,), device=non_sky_depth.device)
+            sampled_depth = non_sky_depth[idx]
+        else:
+            sampled_depth = non_sky_depth
+        non_sky_max = min(torch.quantile(sampled_depth, 0.99), sky_depth_def)
+
+        # Set sky regions to maximum depth and high confidence
+        output.depth, output.depth_conf = set_sky_regions_to_max_depth(
+            output.depth, output.depth_conf, non_sky_mask, max_depth=non_sky_max
+        )
+
+        return output

depth_anything_3/model/dinov2/dinov2.py

@@ -0,0 +1,64 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/main/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py
+
+
+from typing import List
+import torch.nn as nn
+
+from depth_anything_3.model.dinov2.vision_transformer import (
+    vit_base,
+    vit_giant2,
+    vit_large,
+    vit_small,
+)
+
+
+class DinoV2(nn.Module):
+    def __init__(
+        self,
+        name: str,
+        out_layers: List[int],
+        alt_start: int = -1,
+        qknorm_start: int = -1,
+        rope_start: int = -1,
+        cat_token: bool = True,
+        **kwargs,
+    ):
+        super().__init__()
+        assert name in {"vits", "vitb", "vitl", "vitg"}
+        self.name = name
+        self.out_layers = out_layers
+        self.alt_start = alt_start
+        self.qknorm_start = qknorm_start
+        self.rope_start = rope_start
+        self.cat_token = cat_token
+        encoder_map = {
+            "vits": vit_small,
+            "vitb": vit_base,
+            "vitl": vit_large,
+            "vitg": vit_giant2,
+        }
+        encoder_fn = encoder_map[self.name]
+        ffn_layer = "swiglufused" if self.name == "vitg" else "mlp"
+        self.pretrained = encoder_fn(
+            img_size=518,
+            patch_size=14,
+            ffn_layer=ffn_layer,
+            alt_start=alt_start,
+            qknorm_start=qknorm_start,
+            rope_start=rope_start,
+            cat_token=cat_token,
+        )
+
+    def forward(self, x, **kwargs):
+        return self.pretrained.get_intermediate_layers(
+            x,
+            self.out_layers,
+            **kwargs,
+        )

depth_anything_3/model/dinov2/layers/__init__.py

@@ -0,0 +1,25 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# from .attention import MemEffAttention
+from .block import Block
+from .layer_scale import LayerScale
+from .mlp import Mlp
+from .patch_embed import PatchEmbed
+from .rope import PositionGetter, RotaryPositionEmbedding2D
+from .swiglu_ffn import SwiGLUFFN, SwiGLUFFNFused
+
+__all__ = [
+    Mlp,
+    PatchEmbed,
+    SwiGLUFFN,
+    SwiGLUFFNFused,
+    Block,
+    # MemEffAttention,
+    LayerScale,
+    PositionGetter,
+    RotaryPositionEmbedding2D,
+]

depth_anything_3/model/dinov2/layers/attention.py

@@ -0,0 +1,100 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py
+
+import logging
+import torch.nn.functional as F
+from torch import Tensor, nn
+
+logger = logging.getLogger("dinov2")
+
+
+class Attention(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int = 8,
+        qkv_bias: bool = False,
+        proj_bias: bool = True,
+        attn_drop: float = 0.0,
+        proj_drop: float = 0.0,
+        norm_layer: nn.Module = nn.LayerNorm,
+        qk_norm: bool = False,
+        fused_attn: bool = True,  # use F.scaled_dot_product_attention or not
+        rope=None,
+    ) -> None:
+        super().__init__()
+        assert dim % num_heads == 0, "dim should be divisible by num_heads"
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = head_dim**-0.5
+        self.fused_attn = fused_attn
+
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.q_norm = norm_layer(head_dim) if qk_norm else nn.Identity()
+        self.k_norm = norm_layer(head_dim) if qk_norm else nn.Identity()
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim, bias=proj_bias)
+        self.proj_drop = nn.Dropout(proj_drop)
+        self.rope = rope
+
+    def forward(self, x: Tensor, pos=None, attn_mask=None) -> Tensor:
+        B, N, C = x.shape
+        qkv = (
+            self.qkv(x)
+            .reshape(B, N, 3, self.num_heads, C // self.num_heads)
+            .permute(2, 0, 3, 1, 4)
+        )
+        q, k, v = qkv[0], qkv[1], qkv[2]
+        q, k = self.q_norm(q), self.k_norm(k)
+        if self.rope is not None and pos is not None:
+            q = self.rope(q, pos)
+            k = self.rope(k, pos)
+        if self.fused_attn:
+            x = F.scaled_dot_product_attention(
+                q,
+                k,
+                v,
+                dropout_p=self.attn_drop.p if self.training else 0.0,
+                attn_mask=(
+                    (attn_mask)[:, None].repeat(1, self.num_heads, 1, 1)
+                    if attn_mask is not None
+                    else None
+                ),
+            )
+        else:
+            q = q * self.scale
+            attn = q @ k.transpose(-2, -1)
+            attn = attn.softmax(dim=-1)
+            attn = self.attn_drop(attn)
+            x = attn @ v
+
+        x = x.transpose(1, 2).reshape(B, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+    def _forward(self, x: Tensor) -> Tensor:
+        B, N, C = x.shape
+        qkv = (
+            self.qkv(x)
+            .reshape(B, N, 3, self.num_heads, C // self.num_heads)
+            .permute(2, 0, 3, 1, 4)
+        )
+
+        q, k, v = qkv[0] * self.scale, qkv[1], qkv[2]
+        attn = q @ k.transpose(-2, -1)
+
+        attn = attn.softmax(dim=-1)
+        attn = self.attn_drop(attn)
+
+        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x

depth_anything_3/model/dinov2/layers/block.py

@@ -0,0 +1,143 @@
+# flake8: noqa: F821
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py
+
+import logging
+from typing import Callable, Optional
+import torch
+from torch import Tensor, nn
+
+from .attention import Attention
+from .drop_path import DropPath
+from .layer_scale import LayerScale
+from .mlp import Mlp
+
+logger = logging.getLogger("dinov2")
+XFORMERS_AVAILABLE = True
+
+
+class Block(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int,
+        mlp_ratio: float = 4.0,
+        qkv_bias: bool = False,
+        proj_bias: bool = True,
+        ffn_bias: bool = True,
+        drop: float = 0.0,
+        attn_drop: float = 0.0,
+        init_values=None,
+        drop_path: float = 0.0,
+        act_layer: Callable[..., nn.Module] = nn.GELU,
+        norm_layer: Callable[..., nn.Module] = nn.LayerNorm,
+        attn_class: Callable[..., nn.Module] = Attention,
+        ffn_layer: Callable[..., nn.Module] = Mlp,
+        qk_norm: bool = False,
+        rope=None,
+        ln_eps: float = 1e-6,
+    ) -> None:
+        super().__init__()
+        # print(f"biases: qkv: {qkv_bias}, proj: {proj_bias}, ffn: {ffn_bias}")
+        self.norm1 = norm_layer(dim, eps=ln_eps)
+        self.attn = attn_class(
+            dim,
+            num_heads=num_heads,
+            qkv_bias=qkv_bias,
+            proj_bias=proj_bias,
+            attn_drop=attn_drop,
+            proj_drop=drop,
+            qk_norm=qk_norm,
+            rope=rope,
+        )
+        self.ls1 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
+        self.drop_path1 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+
+        self.norm2 = norm_layer(dim, eps=ln_eps)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = ffn_layer(
+            in_features=dim,
+            hidden_features=mlp_hidden_dim,
+            act_layer=act_layer,
+            drop=drop,
+            bias=ffn_bias,
+        )
+        self.ls2 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
+        self.drop_path2 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+
+        self.sample_drop_ratio = drop_path
+
+    def forward(self, x: Tensor, pos=None, attn_mask=None) -> Tensor:
+        def attn_residual_func(x: Tensor, pos=None, attn_mask=None) -> Tensor:
+            return self.ls1(self.attn(self.norm1(x), pos=pos, attn_mask=attn_mask))
+
+        def ffn_residual_func(x: Tensor) -> Tensor:
+            return self.ls2(self.mlp(self.norm2(x)))
+
+        if self.training and self.sample_drop_ratio > 0.1:
+            # the overhead is compensated only for a drop path rate larger than 0.1
+            x = drop_add_residual_stochastic_depth(
+                x,
+                residual_func=attn_residual_func,
+                sample_drop_ratio=self.sample_drop_ratio,
+                pos=pos,
+            )
+            x = drop_add_residual_stochastic_depth(
+                x,
+                residual_func=ffn_residual_func,
+                sample_drop_ratio=self.sample_drop_ratio,
+            )
+        elif self.training and self.sample_drop_ratio > 0.0:
+            x = x + self.drop_path1(attn_residual_func(x, pos=pos, attn_mask=attn_mask))
+            x = x + self.drop_path1(ffn_residual_func(x))  # FIXME: drop_path2
+        else:
+            x = x + attn_residual_func(x, pos=pos, attn_mask=attn_mask)
+            x = x + ffn_residual_func(x)
+        return x
+
+
+def drop_add_residual_stochastic_depth(
+    x: Tensor,
+    residual_func: Callable[[Tensor], Tensor],
+    sample_drop_ratio: float = 0.0,
+    pos: Optional[Tensor] = None,
+) -> Tensor:
+    # 1) extract subset using permutation
+    b, n, d = x.shape
+    sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1)
+    brange = (torch.randperm(b, device=x.device))[:sample_subset_size]
+    x_subset = x[brange]
+
+    # 2) apply residual_func to get residual
+    if pos is not None:
+        # if necessary, apply rope to the subset
+        pos = pos[brange]
+        residual = residual_func(x_subset, pos=pos)
+    else:
+        residual = residual_func(x_subset)
+
+    x_flat = x.flatten(1)
+    residual = residual.flatten(1)
+
+    residual_scale_factor = b / sample_subset_size
+
+    # 3) add the residual
+    x_plus_residual = torch.index_add(
+        x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor
+    )
+    return x_plus_residual.view_as(x)
+
+
+def get_branges_scales(x, sample_drop_ratio=0.0):
+    b, n, d = x.shape
+    sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1)
+    brange = (torch.randperm(b, device=x.device))[:sample_subset_size]
+    residual_scale_factor = b / sample_subset_size
+    return brange, residual_scale_factor

depth_anything_3/model/dinov2/layers/drop_path.py

@@ -0,0 +1,35 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/drop.py
+
+
+from torch import nn
+
+
+def drop_path(x, drop_prob: float = 0.0, training: bool = False):
+    if drop_prob == 0.0 or not training:
+        return x
+    keep_prob = 1 - drop_prob
+    shape = (x.shape[0],) + (1,) * (x.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
+    random_tensor = x.new_empty(shape).bernoulli_(keep_prob)
+    if keep_prob > 0.0:
+        random_tensor.div_(keep_prob)
+    output = x * random_tensor
+    return output
+
+
+class DropPath(nn.Module):
+    """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks)."""
+
+    def __init__(self, drop_prob=None):
+        super().__init__()
+        self.drop_prob = drop_prob
+
+    def forward(self, x):
+        return drop_path(x, self.drop_prob, self.training)

depth_anything_3/model/dinov2/layers/layer_scale.py

@@ -0,0 +1,31 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Modified from: https://github.com/huggingface/pytorch-image-models/blob/main/timm/models/vision_transformer.py#L103-L110  # noqa: E501
+
+from typing import Union
+import torch
+from torch import Tensor, nn
+
+
+class LayerScale(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        init_values: Union[float, Tensor] = 1e-5,
+        inplace: bool = False,
+    ) -> None:
+        super().__init__()
+        self.dim = dim
+        self.inplace = inplace
+        self.init_values = init_values
+        self.gamma = nn.Parameter(init_values * torch.ones(dim))
+
+    def forward(self, x: Tensor) -> Tensor:
+        return x.mul_(self.gamma) if self.inplace else x * self.gamma
+
+    def extra_repr(self) -> str:
+        return f"{self.dim}, init_values={self.init_values}, inplace={self.inplace}"

depth_anything_3/model/dinov2/layers/mlp.py

@@ -0,0 +1,40 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/mlp.py
+
+
+from typing import Callable, Optional
+from torch import Tensor, nn
+
+
+class Mlp(nn.Module):
+    def __init__(
+        self,
+        in_features: int,
+        hidden_features: Optional[int] = None,
+        out_features: Optional[int] = None,
+        act_layer: Callable[..., nn.Module] = nn.GELU,
+        drop: float = 0.0,
+        bias: bool = True,
+    ) -> None:
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features, bias=bias)
+        self.act = act_layer()
+        self.fc2 = nn.Linear(hidden_features, out_features, bias=bias)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x: Tensor) -> Tensor:
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x

depth_anything_3/model/dinov2/layers/patch_embed.py

@@ -0,0 +1,94 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py
+
+from typing import Callable, Optional, Tuple, Union
+import torch.nn as nn
+from torch import Tensor
+
+
+def make_2tuple(x):
+    if isinstance(x, tuple):
+        assert len(x) == 2
+        return x
+
+    assert isinstance(x, int)
+    return (x, x)
+
+
+class PatchEmbed(nn.Module):
+    """
+    2D image to patch embedding: (B,C,H,W) -> (B,N,D)
+
+    Args:
+        img_size: Image size.
+        patch_size: Patch token size.
+        in_chans: Number of input image channels.
+        embed_dim: Number of linear projection output channels.
+        norm_layer: Normalization layer.
+    """
+
+    def __init__(
+        self,
+        img_size: Union[int, Tuple[int, int]] = 224,
+        patch_size: Union[int, Tuple[int, int]] = 16,
+        in_chans: int = 3,
+        embed_dim: int = 768,
+        norm_layer: Optional[Callable] = None,
+        flatten_embedding: bool = True,
+    ) -> None:
+        super().__init__()
+
+        image_HW = make_2tuple(img_size)
+        patch_HW = make_2tuple(patch_size)
+        patch_grid_size = (
+            image_HW[0] // patch_HW[0],
+            image_HW[1] // patch_HW[1],
+        )
+
+        self.img_size = image_HW
+        self.patch_size = patch_HW
+        self.patches_resolution = patch_grid_size
+        self.num_patches = patch_grid_size[0] * patch_grid_size[1]
+
+        self.in_chans = in_chans
+        self.embed_dim = embed_dim
+
+        self.flatten_embedding = flatten_embedding
+
+        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_HW, stride=patch_HW)
+        self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
+
+    def forward(self, x: Tensor) -> Tensor:
+        _, _, H, W = x.shape
+        patch_H, patch_W = self.patch_size
+
+        assert (
+            H % patch_H == 0
+        ), f"Input image height {H} is not a multiple of patch height {patch_H}"
+        assert (
+            W % patch_W == 0
+        ), f"Input image width {W} is not a multiple of patch width: {patch_W}"
+
+        x = self.proj(x)  # B C H W
+        H, W = x.size(2), x.size(3)
+        x = x.flatten(2).transpose(1, 2)  # B HW C
+        x = self.norm(x)
+        if not self.flatten_embedding:
+            x = x.reshape(-1, H, W, self.embed_dim)  # B H W C
+        return x
+
+    def flops(self) -> float:
+        Ho, Wo = self.patches_resolution
+        flops = (
+            Ho * Wo * self.embed_dim * self.in_chans * (self.patch_size[0] * self.patch_size[1])
+        )
+        if self.norm is not None:
+            flops += Ho * Wo * self.embed_dim
+        return flops

depth_anything_3/model/dinov2/layers/rope.py

@@ -0,0 +1,200 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+
+# Implementation of 2D Rotary Position Embeddings (RoPE).
+
+# This module provides a clean implementation of 2D Rotary Position Embeddings,
+# which extends the original RoPE concept to handle 2D spatial positions.
+
+# Inspired by:
+#         https://github.com/meta-llama/codellama/blob/main/llama/model.py
+#         https://github.com/naver-ai/rope-vit
+
+
+from typing import Dict, Tuple
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class PositionGetter:
+    """Generates and caches 2D spatial positions for patches in a grid.
+
+    This class efficiently manages the generation of spatial coordinates for patches
+    in a 2D grid, caching results to avoid redundant computations.
+
+    Attributes:
+        position_cache: Dictionary storing precomputed position tensors for different
+            grid dimensions.
+    """
+
+    def __init__(self):
+        """Initializes the position generator with an empty cache."""
+        self.position_cache: Dict[Tuple[int, int], torch.Tensor] = {}
+
+    def __call__(
+        self, batch_size: int, height: int, width: int, device: torch.device
+    ) -> torch.Tensor:
+        """Generates spatial positions for a batch of patches.
+
+        Args:
+            batch_size: Number of samples in the batch.
+            height: Height of the grid in patches.
+            width: Width of the grid in patches.
+            device: Target device for the position tensor.
+
+        Returns:
+            Tensor of shape (batch_size, height*width, 2) containing y,x coordinates
+            for each position in the grid, repeated for each batch item.
+        """
+        if (height, width) not in self.position_cache:
+            y_coords = torch.arange(height, device=device)
+            x_coords = torch.arange(width, device=device)
+            positions = torch.cartesian_prod(y_coords, x_coords)
+            self.position_cache[height, width] = positions
+
+        cached_positions = self.position_cache[height, width]
+        return cached_positions.view(1, height * width, 2).expand(batch_size, -1, -1).clone()
+
+
+class RotaryPositionEmbedding2D(nn.Module):
+    """2D Rotary Position Embedding implementation.
+
+    This module applies rotary position embeddings to input tokens based on their
+    2D spatial positions. It handles the position-dependent rotation of features
+    separately for vertical and horizontal dimensions.
+
+    Args:
+        frequency: Base frequency for the position embeddings. Default: 100.0
+        scaling_factor: Scaling factor for frequency computation. Default: 1.0
+
+    Attributes:
+        base_frequency: Base frequency for computing position embeddings.
+        scaling_factor: Factor to scale the computed frequencies.
+        frequency_cache: Cache for storing precomputed frequency components.
+    """
+
+    def __init__(self, frequency: float = 100.0, scaling_factor: float = 1.0):
+        """Initializes the 2D RoPE module."""
+        super().__init__()
+        self.base_frequency = frequency
+        self.scaling_factor = scaling_factor
+        self.frequency_cache: Dict[Tuple, Tuple[torch.Tensor, torch.Tensor]] = {}
+
+    def _compute_frequency_components(
+        self, dim: int, seq_len: int, device: torch.device, dtype: torch.dtype
+    ) -> Tuple[torch.Tensor, torch.Tensor]:
+        """Computes frequency components for rotary embeddings.
+
+        Args:
+            dim: Feature dimension (must be even).
+            seq_len: Maximum sequence length.
+            device: Target device for computations.
+            dtype: Data type for the computed tensors.
+
+        Returns:
+            Tuple of (cosine, sine) tensors for frequency components.
+        """
+        cache_key = (dim, seq_len, device, dtype)
+        if cache_key not in self.frequency_cache:
+            # Compute frequency bands
+            exponents = torch.arange(0, dim, 2, device=device).float() / dim
+            inv_freq = 1.0 / (self.base_frequency**exponents)
+
+            # Generate position-dependent frequencies
+            positions = torch.arange(seq_len, device=device, dtype=inv_freq.dtype)
+            angles = torch.einsum("i,j->ij", positions, inv_freq)
+
+            # Compute and cache frequency components
+            angles = angles.to(dtype)
+            angles = torch.cat((angles, angles), dim=-1)
+            cos_components = angles.cos().to(dtype)
+            sin_components = angles.sin().to(dtype)
+            self.frequency_cache[cache_key] = (cos_components, sin_components)
+
+        return self.frequency_cache[cache_key]
+
+    @staticmethod
+    def _rotate_features(x: torch.Tensor) -> torch.Tensor:
+        """Performs feature rotation by splitting and recombining feature dimensions.
+
+        Args:
+            x: Input tensor to rotate.
+
+        Returns:
+            Rotated feature tensor.
+        """
+        feature_dim = x.shape[-1]
+        x1, x2 = x[..., : feature_dim // 2], x[..., feature_dim // 2 :]
+        return torch.cat((-x2, x1), dim=-1)
+
+    def _apply_1d_rope(
+        self,
+        tokens: torch.Tensor,
+        positions: torch.Tensor,
+        cos_comp: torch.Tensor,
+        sin_comp: torch.Tensor,
+    ) -> torch.Tensor:
+        """Applies 1D rotary position embeddings along one dimension.
+
+        Args:
+            tokens: Input token features.
+            positions: Position indices.
+            cos_comp: Cosine components for rotation.
+            sin_comp: Sine components for rotation.
+
+        Returns:
+            Tokens with applied rotary position embeddings.
+        """
+        # Embed positions with frequency components
+        cos = F.embedding(positions, cos_comp)[:, None, :, :]
+        sin = F.embedding(positions, sin_comp)[:, None, :, :]
+        # Apply rotation
+        return (tokens * cos) + (self._rotate_features(tokens) * sin)
+
+    def forward(self, tokens: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
+        """Applies 2D rotary position embeddings to input tokens.
+
+        Args:
+            tokens: Input tensor of shape (batch_size, n_heads, n_tokens, dim).
+                   The feature dimension (dim) must be divisible by 4.
+            positions: Position tensor of shape (batch_size, n_tokens, 2) containing
+                      the y and x coordinates for each token.
+
+        Returns:
+            Tensor of same shape as input with applied 2D rotary position embeddings.
+
+        Raises:
+            AssertionError: If input dimensions are invalid or positions are malformed.
+        """
+        # Validate inputs
+        assert tokens.size(-1) % 2 == 0, "Feature dimension must be even"
+        assert (
+            positions.ndim == 3 and positions.shape[-1] == 2
+        ), "Positions must have shape (batch_size, n_tokens, 2)"
+
+        # Compute feature dimension for each spatial direction
+        feature_dim = tokens.size(-1) // 2
+
+        # Get frequency components
+        max_position = int(positions.max()) + 1
+        cos_comp, sin_comp = self._compute_frequency_components(
+            feature_dim, max_position, tokens.device, tokens.dtype
+        )
+
+        # Split features for vertical and horizontal processing
+        vertical_features, horizontal_features = tokens.chunk(2, dim=-1)
+
+        # Apply RoPE separately for each dimension
+        vertical_features = self._apply_1d_rope(
+            vertical_features, positions[..., 0], cos_comp, sin_comp
+        )
+        horizontal_features = self._apply_1d_rope(
+            horizontal_features, positions[..., 1], cos_comp, sin_comp
+        )
+
+        # Combine processed features
+        return torch.cat((vertical_features, horizontal_features), dim=-1)

depth_anything_3/model/dinov2/layers/swiglu_ffn.py

@@ -0,0 +1,62 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from typing import Callable, Optional
+import torch.nn.functional as F
+from torch import Tensor, nn
+
+
+class SwiGLUFFN(nn.Module):
+    def __init__(
+        self,
+        in_features: int,
+        hidden_features: Optional[int] = None,
+        out_features: Optional[int] = None,
+        act_layer: Callable[..., nn.Module] = None,
+        drop: float = 0.0,
+        bias: bool = True,
+    ) -> None:
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.w12 = nn.Linear(in_features, 2 * hidden_features, bias=bias)
+        self.w3 = nn.Linear(hidden_features, out_features, bias=bias)
+
+    def forward(self, x: Tensor) -> Tensor:
+        x12 = self.w12(x)
+        x1, x2 = x12.chunk(2, dim=-1)
+        hidden = F.silu(x1) * x2
+        return self.w3(hidden)
+
+
+try:
+    from xformers.ops import SwiGLU
+
+    XFORMERS_AVAILABLE = True
+except ImportError:
+    SwiGLU = SwiGLUFFN
+    XFORMERS_AVAILABLE = False
+
+
+class SwiGLUFFNFused(SwiGLU):
+    def __init__(
+        self,
+        in_features: int,
+        hidden_features: Optional[int] = None,
+        out_features: Optional[int] = None,
+        act_layer: Callable[..., nn.Module] = None,
+        drop: float = 0.0,
+        bias: bool = True,
+    ) -> None:
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        hidden_features = (int(hidden_features * 2 / 3) + 7) // 8 * 8
+        super().__init__(
+            in_features=in_features,
+            hidden_features=hidden_features,
+            out_features=out_features,
+            bias=bias,
+        )