data_utils/pyrender_wrapper.py

# Modified from https://github.com/lab4d-org/lab4d

import os
import numpy as np
import cv2
import pyrender
import trimesh
from pyrender import (
    IntrinsicsCamera,
    Mesh,
    Node,
    Scene,
    OffscreenRenderer,
    MetallicRoughnessMaterial,
    RenderFlags
)

os.environ["PYOPENGL_PLATFORM"] = "egl"

def look_at(eye, center, up):
    """Create a look-at (view) matrix."""
    f = np.array(center, dtype=np.float32) - np.array(eye, dtype=np.float32)
    f /= np.linalg.norm(f)

    u = np.array(up, dtype=np.float32)
    u /= np.linalg.norm(u)

    s = np.cross(f, u)
    u = np.cross(s, f)

    m = np.identity(4, dtype=np.float32)
    m[0, :3] = s
    m[1, :3] = u
    m[2, :3] = -f
    m[:3, 3] = -np.matmul(m[:3, :3], np.array(eye, dtype=np.float32))

    return m

class PyRenderWrapper:
    def __init__(self, image_size=(1024, 1024)) -> None:
        # renderer
        self.image_size = image_size
        render_size = max(image_size)
        self.r = OffscreenRenderer(render_size, render_size)
        self.intrinsics = IntrinsicsCamera(
            render_size, render_size, render_size / 2, render_size / 2
        )
        # light
        self.light_pose = np.eye(4)
        self.set_light_topdown()
        self.direc_l = pyrender.DirectionalLight(color=np.ones(3), intensity=5.0)
        self.material = MetallicRoughnessMaterial(
            roughnessFactor=0.75, metallicFactor=0.75, alphaMode="BLEND"
        )
        self.init_camera()

    def init_camera(self):
        self.flip_pose = np.eye(4)
        self.set_camera(np.eye(4))

    def set_camera(self, scene_to_cam):
        # object to camera transforms
        self.scene_to_cam = self.flip_pose @ scene_to_cam
        
    def set_light_topdown(self, gl=False):
        # top down light, slightly closer to the camera
        if gl:
            rot = cv2.Rodrigues(np.asarray([-np.pi / 2, 0, 0]))[0]
        else:
            rot = cv2.Rodrigues(np.asarray([np.pi / 2, 0, 0]))[0]
        self.light_pose[:3, :3] = rot

    def align_light_to_camera(self):
        self.light_pose = np.linalg.inv(self.scene_to_cam)

    def set_intrinsics(self, intrinsics):
        """
        Args:
            intrinsics: (4,) fx,fy,px,py
        """
        self.intrinsics = IntrinsicsCamera(
            intrinsics[0], intrinsics[1], intrinsics[2], intrinsics[3]
        )

    def get_cam_to_scene(self):
        cam_to_scene = np.eye(4)
        cam_to_scene[:3, :3] = self.scene_to_cam[:3, :3].T
        cam_to_scene[:3, 3] = -self.scene_to_cam[:3, :3].T @ self.scene_to_cam[:3, 3]
        return cam_to_scene
    
    def set_camera_view(self, angle, bbox_center, distance=2.0):
        # Calculate camera position based on angle and distance from bounding box center
        camera_position = bbox_center + distance * np.array([np.sin(angle), 0, np.cos(angle)], dtype=np.float32)
        look_at_matrix = look_at(camera_position, bbox_center, [0, 1, 0])
        self.scene_to_cam = look_at_matrix @ self.flip_pose
    
    def render(self, input_dict):
        # Create separate scenes for transparent objects (mesh) and solid objects (joints and bones)
        scene_transparent = Scene(ambient_light=np.array([1.0, 1.0, 1.0, 1.0]) * 0.1)
        scene_solid = Scene(ambient_light=np.array([1.0, 1.0, 1.0, 1.0]) * 0.1)

        mesh_pyrender = Mesh.from_trimesh(input_dict["shape"], smooth=False)
        mesh_pyrender.primitives[0].material = self.material
        scene_transparent.add(mesh_pyrender, pose=np.eye(4), name="shape")

        if "joint_meshes" in input_dict:
            joints_pyrender = Mesh.from_trimesh(input_dict["joint_meshes"], smooth=False)
            joints_pyrender.primitives[0].material = self.material
            scene_solid.add(joints_pyrender, pose=np.eye(4), name="joints")

        if "bone_meshes" in input_dict:
            bones_pyrender = Mesh.from_trimesh(input_dict["bone_meshes"], smooth=False)
            bones_pyrender.primitives[0].material = self.material
            scene_solid.add(bones_pyrender, pose=np.eye(4), name="bones")

        # Camera for both scenes
        scene_transparent.add(self.intrinsics, pose=self.get_cam_to_scene())
        scene_solid.add(self.intrinsics, pose=self.get_cam_to_scene())

        # Light for both scenes
        scene_transparent.add(self.direc_l, pose=self.light_pose)
        scene_solid.add(self.direc_l, pose=self.light_pose)

        # Render transparent scene first
        color_transparent, depth_transparent = self.r.render(scene_transparent)

        # Render solid scene on top
        color_solid, depth_solid = self.r.render(scene_solid)

        # Combine the two scenes
        color_combined = np.where(depth_solid[..., np.newaxis] == 0, color_transparent, color_solid)

        return color_combined, depth_solid
    def delete(self):
        self.r.delete()