evaluation/utils/point_cloud_converter.py

"""
Helper to convert depth images (millimeters) to point clouds (meters)
using Open3D and the same pinhole camera model as pc_offiline.py
(verified accuracy). Intrinsics are scaled when depth resolution
differs from the reference 1280x720.
"""

import numpy as np

try:
    import open3d as o3d
except ImportError:
    o3d = None


class PointCloudConverter:
    """
    Converts depth images (mm) to point clouds (m) using Open3D's
    create_from_depth_image and pinhole model (same as pc_offiline.py).
    Intrinsics are for the rectified left ZED camera at 1280x720.
    """

    def __init__(self):
        # ZED intrinsics at reference resolution 1280x720 (from calibrator.K_zed)
        self._K = np.array(
            [
                [521.581604, 0.0, 636.33398438],
                [0.0, 521.581604, 373.10964966],
                [0.0, 0.0, 1.0],
            ],
            dtype=np.float64,
        )
        self._ref_w = 1280
        self._ref_h = 720

    def scale_intrinsics(self, width: int, height: int) -> np.ndarray:
        """
        Scale intrinsics to a different image size (e.g. when depth is not 1280x720).
        Same scaling as pc_offiline (fx, fy, cx, cy scaled by width/ref_w and height/ref_h).

        Parameters:
            width: Image width (pixels).
            height: Image height (pixels).

        Returns:
            K: 3x3 intrinsic matrix for the given resolution, dtype float64.
        """
        sx = width / self._ref_w
        sy = height / self._ref_h
        K_scaled = np.array(
            [
                [self._K[0, 0] * sx, 0.0, self._K[0, 2] * sx],
                [0.0, self._K[1, 1] * sy, self._K[1, 2] * sy],
                [0.0, 0.0, 1.0],
            ],
            dtype=np.float64,
        )
        return K_scaled

    def depth_pixel_to_point(self, u: float, v: float, depth_m: float, width: int, height: int) -> np.ndarray:
        """
        Back-project one depth pixel into metric camera coordinates.

        Returns [x_right, y_down, z_forward] in meters.
        """
        K = self.scale_intrinsics(width, height)
        fx, fy = K[0, 0], K[1, 1]
        cx, cy = K[0, 2], K[1, 2]
        x = (float(u) - cx) * float(depth_m) / fx
        y = (float(v) - cy) * float(depth_m) / fy
        z = float(depth_m)
        return np.array([x, y, z], dtype=np.float32)

    def project_point_cloud_to_depth_image(
        self,
        points_xyz: np.ndarray,
        width: int,
        height: int,
        depth_scale: float = 1000.0,
        depth_max: float = 10.6,
    ) -> np.ndarray:
        """
        Project camera-centered points [x_right, y_up, z_forward] to a depth image.

        Returns a float32 depth image in meters on the requested image grid.
        """
        if o3d is None:
            raise ImportError("open3d is required for project_point_cloud_to_depth_image but is not installed")

        if points_xyz.size == 0:
            return np.zeros((height, width), dtype=np.float32)

        points_cam = np.asarray(points_xyz, dtype=np.float32).copy()
        points_cam[:, 1] *= -1.0  # Open3D pinhole projection uses image-space y-down.

        pcd = o3d.t.geometry.PointCloud()
        pcd.point["positions"] = o3d.core.Tensor(points_cam, dtype=o3d.core.Dtype.Float32)

        intrinsics = o3d.core.Tensor(self.scale_intrinsics(width, height).astype(np.float32))
        extrinsics = o3d.core.Tensor.eye(4, o3d.core.Dtype.Float32)
        depth = pcd.project_to_depth_image(
            width=width,
            height=height,
            intrinsics=intrinsics,
            extrinsics=extrinsics,
            depth_scale=float(depth_scale),
            depth_max=float(depth_max),
        )
        depth_np = depth.as_tensor().numpy()
        depth_np = np.asarray(depth_np).squeeze().astype(np.float32)
        if depth_np.size == 0:
            return np.zeros((height, width), dtype=np.float32)

        if np.nanmax(depth_np) > depth_max + 1.0:
            depth_np = depth_np / float(depth_scale)
        return depth_np.astype(np.float32)

    def depth_to_point_cloud(
        self,
        depth_mm: np.ndarray,
        depth_scale: float = 1000.0,
        max_depth_meters: float = 11.2,
    ) -> object:
        """
        Convert depth image to point cloud using Open3D's pinhole back-projection
        (same method as pc_offiline.depth_to_point_cloud_open3d). Depth in millimeters,
        output points in meters.

        Parameters:
            depth_mm: Depth image, shape (H, W), dtype float (millimeters).
                      Invalid depth (0, NaN, inf, or > depth_trunc) is skipped.

        Returns:
            Open3D PointCloud with points in meters. Uses the same
            depth_scale=depth_scale and depth_trunc=max_depth_meters.
        """
        if o3d is None:
            raise ImportError("open3d is required for depth_to_point_cloud but is not installed")

        h, w = depth_mm.shape[:2]
        K = self.scale_intrinsics(w, h)
        fx, fy = K[0, 0], K[1, 1]
        cx, cy = K[0, 2], K[1, 2]

        o3d_intrinsic = o3d.camera.PinholeCameraIntrinsic(
            width=w,
            height=h,
            fx=fx,
            fy=fy,
            cx=cx,
            cy=cy,
        )
        depth_o3d = o3d.geometry.Image(depth_mm.astype(np.float32))
        pcd = o3d.geometry.PointCloud.create_from_depth_image(
            depth_o3d,
            o3d_intrinsic,  
            depth_scale=depth_scale,
            depth_trunc=max_depth_meters,
            project_valid_depth_only=True,
        )
        return pcd