added PPD models

.github/workflows/huggingface.yml

@@ -17,9 +17,9 @@ jobs:
         env:
           HF: ${{secrets.HF_TOKEN }}
           HFUSER: ${{secrets.HFUSER }}
-        run: git remote add space https://$HFUSER:$HF@huggingface.co/spaces/$HFUSER/Depth-Anything-Compare-demo
-      - name: Push to hub
+        run: git remote add space https://$HFUSER:$HF@huggingface.co/spaces/$HFUSER/Depth-Estimation-Compare-demo
+      - name: Push to huggingface hub
         env:
           HF: ${{ secrets.HF_TOKEN}}
           HFUSER: ${{secrets.HFUSER }}
-        run: git push --force https://$HFUSER:$HF@huggingface.co/spaces/$HFUSER/Depth-Anything-Compare-demo main
+        run: git push --force https://$HFUSER:$HF@huggingface.co/spaces/$HFUSER/Depth-Estimation-Compare-demo main

Pixel-Perfect-Depth/.gitattributes

@@ -0,0 +1,54 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/examples/0001.jpg filter=lfs diff=lfs merge=lfs -text
+assets/examples/0003.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/0004.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/0005.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/0006.jpg filter=lfs diff=lfs merge=lfs -text
+assets/examples/0007.jpg filter=lfs diff=lfs merge=lfs -text
+assets/examples/0008.jpg filter=lfs diff=lfs merge=lfs -text
+assets/examples/0009.jpg filter=lfs diff=lfs merge=lfs -text
+assets/examples/0010.jpg filter=lfs diff=lfs merge=lfs -text
+assets/examples/0004.jpg filter=lfs diff=lfs merge=lfs -text
+assets/examples/0005.jpg filter=lfs diff=lfs merge=lfs -text
+assets/examples/0011.jpg filter=lfs diff=lfs merge=lfs -text
+assets/examples/0001.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/0002.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/0003.JPG filter=lfs diff=lfs merge=lfs -text
+assets/examples/0006.PNG filter=lfs diff=lfs merge=lfs -text
+assets/examples/0007.PNG filter=lfs diff=lfs merge=lfs -text
+assets/examples/0008.PNG filter=lfs diff=lfs merge=lfs -text
+assets/examples/0009.PNG filter=lfs diff=lfs merge=lfs -text

Pixel-Perfect-Depth/app.py

@@ -0,0 +1,209 @@
+import gradio as gr
+import cv2
+import matplotlib
+import numpy as np
+import os
+import time
+from PIL import Image
+import torch
+import torch.nn.functional as F
+import open3d as o3d
+import trimesh
+import tempfile
+import shutil
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
+from gradio_imageslider import ImageSlider
+from huggingface_hub import hf_hub_download
+
+from ppd.utils.set_seed import set_seed
+from ppd.utils.align_depth_func import recover_metric_depth_ransac
+from ppd.utils.depth2pcd import depth2pcd
+from moge.model.v2 import MoGeModel 
+from ppd.models.ppd import PixelPerfectDepth
+
+try:
+    import spaces
+    HUGGINFACE_SPACES_INSTALLED = True
+except ImportError:
+    HUGGINFACE_SPACES_INSTALLED = False
+
+css = """
+#img-display-container {
+    max-height: 100vh;
+}
+#img-display-input {
+    max-height: 100vh;
+}
+#img-display-output {
+    max-height: 100vh;
+}
+#download {
+    height: 62px;
+}
+
+#img-display-output .image-slider-image {
+    object-fit: contain !important;
+    width: 100% !important;
+    height: 100% !important;
+}
+"""
+
+set_seed(666)
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+default_steps = 20
+model = PixelPerfectDepth(sampling_steps=default_steps)
+ckpt_path = hf_hub_download(
+    repo_id="gangweix/Pixel-Perfect-Depth",
+    filename="ppd.pth",
+    repo_type="model"
+)
+state_dict = torch.load(ckpt_path, map_location="cpu")
+model.load_state_dict(state_dict, strict=False)
+model = model.eval()
+model = model.to(DEVICE)
+
+moge_model = MoGeModel.from_pretrained("Ruicheng/moge-2-vitl-normal").eval()
+moge_model = moge_model.to(DEVICE)
+
+
+def main(share=True):
+    print("Initializing Pixel-Perfect Depth Demo...")
+
+    cmap = matplotlib.colormaps.get_cmap('Spectral')
+
+    title = "# Pixel-Perfect Depth"
+    description = """Official demo for **Pixel-Perfect Depth**.
+    Please refer to our [paper](https://arxiv.org/pdf/2510.07316), [project page](https://pixel-perfect-depth.github.io), and [github](https://github.com/gangweix/pixel-perfect-depth) for more details."""
+
+    @(spaces.GPU if HUGGINFACE_SPACES_INSTALLED else (lambda x: x))
+    def predict_depth(image, denoise_steps):
+        depth, resize_image = model.infer_image(image, sampling_steps=denoise_steps)
+        return depth, resize_image
+
+    @(spaces.GPU if HUGGINFACE_SPACES_INSTALLED else (lambda x: x))
+    def predict_moge_depth(image):
+        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        image = torch.tensor(image / 255, dtype=torch.float32, device=DEVICE).permute(2, 0, 1)
+        metric_depth, mask, intrinsics = moge_model.infer(image)
+        metric_depth[~mask] = metric_depth[mask].max()
+        return metric_depth, mask, intrinsics
+
+    def on_submit(image, denoise_steps, apply_filter, request: gr.Request = None):
+
+        H, W = image.shape[:2]
+        ppd_depth, resize_image = predict_depth(image[:, :, ::-1], denoise_steps)
+        resize_H, resize_W = resize_image.shape[:2]
+
+        # moge provide metric depth and intrinsics
+        moge_depth, mask, intrinsics = predict_moge_depth(resize_image)
+
+        # relative depth -> metric depth
+        metric_depth = recover_metric_depth_ransac(ppd_depth, moge_depth, mask)
+        intrinsics[0, 0] *= resize_W 
+        intrinsics[1, 1] *= resize_H
+        intrinsics[0, 2] *= resize_W
+        intrinsics[1, 2] *= resize_H
+
+        # metric depth -> point cloud
+        pcd = depth2pcd(metric_depth, intrinsics, color=cv2.cvtColor(resize_image, cv2.COLOR_BGR2RGB), input_mask=mask, ret_pcd=True)
+        if apply_filter:
+            cl, ind = pcd.remove_statistical_outlier(nb_neighbors=20, std_ratio=2.0)
+            pcd = pcd.select_by_index(ind)
+
+        tempdir = Path(tempfile.gettempdir(), 'ppd')
+        tempdir.mkdir(exist_ok=True)
+        output_path = Path(tempdir, request.session_hash)
+        shutil.rmtree(output_path, ignore_errors=True)
+        output_path.mkdir(exist_ok=True, parents=True)
+        
+        ply_path = os.path.join(output_path, 'pointcloud.ply')
+
+        # save pcd to temporary .ply
+        pcd.points = o3d.utility.Vector3dVector(
+            np.asarray(pcd.points) * np.array([1, -1, -1], dtype=np.float32)
+        )
+        o3d.io.write_point_cloud(ply_path, pcd)
+        vertices = np.asarray(pcd.points)
+        vertex_colors = (np.asarray(pcd.colors) * 255).astype(np.uint8)
+        mesh = trimesh.PointCloud(vertices=vertices, colors=vertex_colors)
+        glb_path = os.path.join(output_path, 'pointcloud.glb')
+        mesh.export(glb_path)
+
+
+        # save raw depth (npy)
+        depth = cv2.resize(ppd_depth, (W, H), interpolation=cv2.INTER_LINEAR)
+        raw_depth_path = os.path.join(output_path, 'raw_depth.npy')
+        np.save(raw_depth_path, depth)
+
+        depth_vis = (depth - depth.min()) / (depth.max() - depth.min() + 1e-5) * 255.0
+        depth_vis = depth_vis.astype(np.uint8)
+        colored_depth = (cmap(depth_vis)[:, :, :3] * 255).astype(np.uint8)
+
+        split_region = np.ones((image.shape[0], 50, 3), dtype=np.uint8) * 255
+        combined_result = cv2.hconcat([image[:, :, ::-1], split_region, colored_depth[:, :, ::-1]])
+
+        vis_path = os.path.join(output_path, 'image_depth_vis.png')
+        cv2.imwrite(vis_path, combined_result)
+    
+        file_names = ["image_depth_vis.png", "raw_depth.npy", "pointcloud.ply"]
+            
+        download_files = [
+            (output_path / name).as_posix()
+            for name in file_names
+            if (output_path / name).exists()
+        ]
+
+        return [(image, colored_depth), glb_path, download_files]
+
+
+    with gr.Blocks(theme=gr.themes.Soft()) as demo:
+        gr.Markdown(title)
+        gr.Markdown(description)
+        gr.Markdown("### Point Cloud & Depth Prediction demo")
+
+        with gr.Row():
+            # Left: input image + settings
+            with gr.Column():
+                input_image = gr.Image(label="Input Image", image_mode="RGB", type='numpy', elem_id='img-display-input')
+                with gr.Accordion(label="Settings", open=False):
+                    denoise_steps = gr.Slider(label="Denoising Steps", minimum=1, maximum=100, value=20, step=1)
+                    apply_filter = gr.Checkbox(label="Apply filter points", value=True)
+                submit_btn = gr.Button(value="Predict")
+
+            # Right: 3D point cloud + depth
+            with gr.Column():
+                with gr.Tabs():
+                    with gr.Tab("3D View"):
+                        model_3d = gr.Model3D(display_mode="solid", label="3D Point Map", clear_color=[1,1,1,1], height="60vh")
+                    with gr.Tab("Depth"):
+                        depth_map = ImageSlider(label="Depth Map with Slider View", elem_id='img-display-output', position=0.5)
+                    with gr.Tab("Download"):
+                        download_files = gr.File(type='filepath', label="Download Files")
+
+        submit_btn.click(
+            fn=lambda: [None, None, None, "", "", ""],
+            outputs=[depth_map, model_3d, download_files]
+        ).then(
+            fn=on_submit,
+            inputs=[input_image, denoise_steps, apply_filter],
+            outputs=[depth_map, model_3d, download_files]
+        )
+
+        example_files = os.listdir('assets/examples')
+        example_files.sort()
+        example_files = [os.path.join('assets/examples', filename) for filename in example_files]
+        examples = gr.Examples(
+            examples=example_files, 
+            inputs=input_image, 
+            outputs=[depth_map, model_3d, download_files], 
+            fn=on_submit,
+            cache_examples=False
+        )
+        
+    demo.queue().launch(share=share) 
+
+if __name__ == '__main__':
+    main(share=True)

Pixel-Perfect-Depth/moge/model/__init__.py

@@ -0,0 +1,18 @@
+import importlib
+from typing import *
+
+if TYPE_CHECKING:
+    from .v1 import MoGeModel as MoGeModelV1
+    from .v2 import MoGeModel as MoGeModelV2
+
+
+def import_model_class_by_version(version: str) -> Type[Union['MoGeModelV1', 'MoGeModelV2']]:
+    assert version in ['v1', 'v2'], f'Unsupported model version: {version}'
+    
+    try:
+        module = importlib.import_module(f'.{version}', __package__)
+    except ModuleNotFoundError:
+        raise ValueError(f'Model version "{version}" not found.')
+
+    cls = getattr(module, 'MoGeModel')
+    return cls

Pixel-Perfect-Depth/moge/model/dinov2/__init__.py

@@ -0,0 +1,6 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+__version__ = "0.0.1"

Pixel-Perfect-Depth/moge/model/dinov2/hub/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.

Pixel-Perfect-Depth/moge/model/dinov2/hub/backbones.py

@@ -0,0 +1,156 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+from enum import Enum
+from typing import Union
+
+import torch
+
+from .utils import _DINOV2_BASE_URL, _make_dinov2_model_name
+
+
+class Weights(Enum):
+    LVD142M = "LVD142M"
+
+
+def _make_dinov2_model(
+    *,
+    arch_name: str = "vit_large",
+    img_size: int = 518,
+    patch_size: int = 14,
+    init_values: float = 1.0,
+    ffn_layer: str = "mlp",
+    block_chunks: int = 0,
+    num_register_tokens: int = 0,
+    interpolate_antialias: bool = False,
+    interpolate_offset: float = 0.1,
+    pretrained: bool = True,
+    weights: Union[Weights, str] = Weights.LVD142M,
+    **kwargs,
+):
+    from ..models import vision_transformer as vits
+
+    if isinstance(weights, str):
+        try:
+            weights = Weights[weights]
+        except KeyError:
+            raise AssertionError(f"Unsupported weights: {weights}")
+
+    model_base_name = _make_dinov2_model_name(arch_name, patch_size)
+    vit_kwargs = dict(
+        img_size=img_size,
+        patch_size=patch_size,
+        init_values=init_values,
+        ffn_layer=ffn_layer,
+        block_chunks=block_chunks,
+        num_register_tokens=num_register_tokens,
+        interpolate_antialias=interpolate_antialias,
+        interpolate_offset=interpolate_offset,
+    )
+    vit_kwargs.update(**kwargs)
+    model = vits.__dict__[arch_name](**vit_kwargs)
+
+    if pretrained:
+        model_full_name = _make_dinov2_model_name(arch_name, patch_size, num_register_tokens)
+        url = _DINOV2_BASE_URL + f"/{model_base_name}/{model_full_name}_pretrain.pth"
+        state_dict = torch.hub.load_state_dict_from_url(url, map_location="cpu")
+        model.load_state_dict(state_dict, strict=True)
+
+    return model
+
+
+def dinov2_vits14(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs):
+    """
+    DINOv2 ViT-S/14 model (optionally) pretrained on the LVD-142M dataset.
+    """
+    return _make_dinov2_model(arch_name="vit_small", pretrained=pretrained, weights=weights, **kwargs)
+
+
+def dinov2_vitb14(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs):
+    """
+    DINOv2 ViT-B/14 model (optionally) pretrained on the LVD-142M dataset.
+    """
+    return _make_dinov2_model(arch_name="vit_base", pretrained=pretrained, weights=weights, **kwargs)
+
+
+def dinov2_vitl14(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs):
+    """
+    DINOv2 ViT-L/14 model (optionally) pretrained on the LVD-142M dataset.
+    """
+    return _make_dinov2_model(arch_name="vit_large", pretrained=pretrained, weights=weights, **kwargs)
+
+
+def dinov2_vitg14(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs):
+    """
+    DINOv2 ViT-g/14 model (optionally) pretrained on the LVD-142M dataset.
+    """
+    return _make_dinov2_model(
+        arch_name="vit_giant2",
+        ffn_layer="swiglufused",
+        weights=weights,
+        pretrained=pretrained,
+        **kwargs,
+    )
+
+
+def dinov2_vits14_reg(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs):
+    """
+    DINOv2 ViT-S/14 model with registers (optionally) pretrained on the LVD-142M dataset.
+    """
+    return _make_dinov2_model(
+        arch_name="vit_small",
+        pretrained=pretrained,
+        weights=weights,
+        num_register_tokens=4,
+        interpolate_antialias=True,
+        interpolate_offset=0.0,
+        **kwargs,
+    )
+
+
+def dinov2_vitb14_reg(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs):
+    """
+    DINOv2 ViT-B/14 model with registers (optionally) pretrained on the LVD-142M dataset.
+    """
+    return _make_dinov2_model(
+        arch_name="vit_base",
+        pretrained=pretrained,
+        weights=weights,
+        num_register_tokens=4,
+        interpolate_antialias=True,
+        interpolate_offset=0.0,
+        **kwargs,
+    )
+
+
+def dinov2_vitl14_reg(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs):
+    """
+    DINOv2 ViT-L/14 model with registers (optionally) pretrained on the LVD-142M dataset.
+    """
+    return _make_dinov2_model(
+        arch_name="vit_large",
+        pretrained=pretrained,
+        weights=weights,
+        num_register_tokens=4,
+        interpolate_antialias=True,
+        interpolate_offset=0.0,
+        **kwargs,
+    )
+
+
+def dinov2_vitg14_reg(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs):
+    """
+    DINOv2 ViT-g/14 model with registers (optionally) pretrained on the LVD-142M dataset.
+    """
+    return _make_dinov2_model(
+        arch_name="vit_giant2",
+        ffn_layer="swiglufused",
+        weights=weights,
+        pretrained=pretrained,
+        num_register_tokens=4,
+        interpolate_antialias=True,
+        interpolate_offset=0.0,
+        **kwargs,
+    )

Pixel-Perfect-Depth/moge/model/dinov2/hub/utils.py

@@ -0,0 +1,39 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+import itertools
+import math
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+_DINOV2_BASE_URL = "https://dl.fbaipublicfiles.com/dinov2"
+
+
+def _make_dinov2_model_name(arch_name: str, patch_size: int, num_register_tokens: int = 0) -> str:
+    compact_arch_name = arch_name.replace("_", "")[:4]
+    registers_suffix = f"_reg{num_register_tokens}" if num_register_tokens else ""
+    return f"dinov2_{compact_arch_name}{patch_size}{registers_suffix}"
+
+
+class CenterPadding(nn.Module):
+    def __init__(self, multiple):
+        super().__init__()
+        self.multiple = multiple
+
+    def _get_pad(self, size):
+        new_size = math.ceil(size / self.multiple) * self.multiple
+        pad_size = new_size - size
+        pad_size_left = pad_size // 2
+        pad_size_right = pad_size - pad_size_left
+        return pad_size_left, pad_size_right
+
+    @torch.inference_mode()
+    def forward(self, x):
+        pads = list(itertools.chain.from_iterable(self._get_pad(m) for m in x.shape[:1:-1]))
+        output = F.pad(x, pads)
+        return output

Pixel-Perfect-Depth/moge/model/dinov2/layers/__init__.py

@@ -0,0 +1,11 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+from .dino_head import DINOHead
+from .mlp import Mlp
+from .patch_embed import PatchEmbed
+from .swiglu_ffn import SwiGLUFFN, SwiGLUFFNFused
+from .block import NestedTensorBlock
+from .attention import MemEffAttention