mapnurec/model.py

"""MapNuRec — per-pixel feed-forward 3DGS, InstantNuRec-style, warm-started from
Depth-Anything-V2. Per input view i: DA-V2 predicts relative inverse-depth (disparity)
`disp`; a learned global affine maps it to metric depth `z = 1/(a*disp + b)` (a,b>0),
so DA-V2's strong relative geometry is inherited immediately and only the metric scale
is learned (anchored by the map-depth loss on ground). A small fresh per-pixel head on
[rgb, disp] predicts opacity / log-scale-mult / rotation; color = the source pixel RGB.
Each pixel is lifted to a world-space Gaussian; the union over views is rendered by gsplat.
"""
from __future__ import annotations

import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision import transforms

IMAGENET = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])


def lift_to_world(z, K, c2w):
    """z [V,H,W] metric depth (along optical axis) -> world points [V,H,W,3] (OpenCV +z fwd)."""
    V, H, W = z.shape
    dev = z.device
    ys, xs = torch.meshgrid(torch.arange(H, device=dev), torch.arange(W, device=dev), indexing="ij")
    u, v = (xs + 0.5).float(), (ys + 0.5).float()
    fx, fy = K[:, 0, 0, None, None], K[:, 1, 1, None, None]
    cx, cy = K[:, 0, 2, None, None], K[:, 1, 2, None, None]
    x = (u - cx) / fx * z
    y = (v - cy) / fy * z
    p_cam = torch.stack([x, y, z], dim=-1)                               # [V,H,W,3]
    R, t = c2w[:, :3, :3], c2w[:, :3, 3]
    return torch.einsum("vij,vhwj->vhwi", R, p_cam) + t[:, None, None, :]


class MapNuRec(nn.Module):
    def __init__(self, depth_min=2.0, depth_max=80.0):
        super().__init__()
        from transformers import AutoModelForDepthEstimation
        self.da = AutoModelForDepthEstimation.from_pretrained(
            "depth-anything/Depth-Anything-V2-Small-hf")               # DINOv2-S + DPT (warm-start)
        # disparity -> metric depth affine z = 1/(softplus(a)*disp + softplus(b)).
        # init from DA-V2 driving-disp stats (median~2.5): a=softplus(-3.8)=0.022,
        # b=softplus(-4.6)=0.010 -> z(med)~15m, z(horizon,disp~0)~100m. Matches the map
        # scale so the gamma-weighted map-depth anchor is active from step 0 (not zeroed).
        self.aff = nn.Parameter(torch.tensor([-3.8, -4.6]))
        self.dmin, self.dmax = depth_min, depth_max
        # fresh per-pixel head on [rgb(3), disp(1)] -> opacity(1), log_scale_mult(1), rot(4)
        self.head = nn.Sequential(
            nn.Conv2d(4, 64, 3, padding=1), nn.GELU(),
            nn.Conv2d(64, 64, 3, padding=1), nn.GELU(),
            nn.Conv2d(64, 9, 1))
        nn.init.zeros_(self.head[-1].weight); nn.init.zeros_(self.head[-1].bias)
        # channel layout: [0]=opacity, [1:4]=log-scale (ANISOTROPIC), [4:8]=rot quat, [8]=spare
        self.head[-1].bias.data[0] = 2.0                               # opacity logit -> sigmoid~0.88
        self.head[-1].bias.data[4] = 1.0                               # rot quat w=1 (identity)

    def disp(self, images):
        """images [V,3,H,W] in [0,1] -> per-pixel disparity [V,H,W] (larger=closer)."""
        x = IMAGENET(images)
        out = self.da(pixel_values=x).predicted_depth
        if out.dim() == 4:
            out = out[:, 0]
        if out.shape[-2:] != images.shape[-2:]:
            out = F.interpolate(out[:, None], size=images.shape[-2:], mode="bilinear",
                                align_corners=False)[:, 0]
        return out.clamp(min=1e-3)

    def forward(self, images, K, c2w):
        """images [V,3,H,W] (0..1), K [V,3,3], c2w [V,4,4] -> gaussian dict (world frame)."""
        V, _, H, W = images.shape
        disp = self.disp(images)                                       # [V,H,W]
        a, b = F.softplus(self.aff[0]), F.softplus(self.aff[1])
        z = (1.0 / (a * disp + b)).clamp(self.dmin, self.dmax)         # metric depth
        h = self.head(torch.cat([images, disp[:, None]], dim=1))       # [V,9,H,W]
        h = h.permute(0, 2, 3, 1)                                      # [V,H,W,9]
        opacity = torch.sigmoid(h[..., 0])                            # [V,H,W]
        base = (z / K[:, 0, 0, None, None]).clamp(min=1e-4)           # pixel footprint at depth
        scale = base[..., None] * torch.exp(h[..., 1:4].clamp(-3, 3))  # [V,H,W,3] anisotropic
        quat = F.normalize(h[..., 4:8] + torch.tensor([1.0, 0, 0, 0], device=images.device), dim=-1)
        xyz = lift_to_world(z, K, c2w)                                 # [V,H,W,3]
        rgb = images.permute(0, 2, 3, 1)                              # color = source pixel
        flat = lambda t, c: t.reshape(-1, c) if c > 1 else t.reshape(-1)
        return dict(means=xyz.reshape(-1, 3), scales=scale.reshape(-1, 3),
                    quats=quat.reshape(-1, 4), opacities=opacity.reshape(-1),
                    colors=rgb.reshape(-1, 3), depth=z)