engine/pose_estimation/pose_utils/render.py

import os
import imageio
import numpy as np
import torch
from tqdm import tqdm

from pytorch3d.renderer import (
    PerspectiveCameras,
    TexturesVertex,
    PointLights,
    Materials,
    RasterizationSettings,
    MeshRenderer,
    MeshRasterizer,
    SoftPhongShader,
)
from pytorch3d.renderer.mesh.shader import ShaderBase
from pytorch3d.structures import Meshes


class NormalShader(ShaderBase):
    def __init__(self, device="cpu", **kwargs):
        super().__init__(device=device, **kwargs)

    def forward(self, fragments, meshes, **kwargs):
        blend_params = kwargs.get("blend_params", self.blend_params)
        texels = fragments.bary_coords.clone()
        texels = texels.permute(0, 3, 1, 2, 4)
        texels = texels * 2 - 1  # 将 bary_coords 映射到 [-1, 1]

        # 获取法线
        verts_normals = meshes.verts_normals_packed()
        faces_normals = verts_normals[meshes.faces_packed()]
        bary_coords = fragments.bary_coords

        pixel_normals = (
            bary_coords[..., None] * faces_normals[fragments.pix_to_face]
        ).sum(dim=-2)
        pixel_normals = pixel_normals / pixel_normals.norm(dim=-1, keepdim=True)

        # 将法线映射到颜色空间
        # colors = (pixel_normals + 1) / 2  # 将法线映射到 [0, 1]
        colors = torch.clamp(pixel_normals, -1, 1)
        print(colors.shape)
        mask = (fragments.pix_to_face > 0).float()
        colors = torch.cat([colors, mask.unsqueeze(-1)], dim=-1)
        # colors[fragments.pix_to_face < 0] = 0

        # 混合颜色
        # images = self.blend(texels, colors, fragments, blend_params)
        return colors


def overlay_image_onto_background(image, mask, bbox, background):
    if isinstance(image, torch.Tensor):
        image = image.detach().cpu().numpy()
    if isinstance(mask, torch.Tensor):
        mask = mask.detach().cpu().numpy()

    out_image = background.copy()
    bbox = bbox[0].int().cpu().numpy().copy()
    roi_image = out_image[bbox[1] : bbox[3], bbox[0] : bbox[2]]
    if len(roi_image) < 1 or len(roi_image[1]) < 1:
        return out_image
    try:
        roi_image[mask] = image[mask]
    except Exception as e:
        raise e
    out_image[bbox[1] : bbox[3], bbox[0] : bbox[2]] = roi_image

    return out_image


def update_intrinsics_from_bbox(K_org, bbox):
    """
    update intrinsics for cropped images
    """
    device, dtype = K_org.device, K_org.dtype

    K = torch.zeros((K_org.shape[0], 4, 4)).to(device=device, dtype=dtype)
    K[:, :3, :3] = K_org.clone()
    K[:, 2, 2] = 0
    K[:, 2, -1] = 1
    K[:, -1, 2] = 1

    image_sizes = []
    for idx, bbox in enumerate(bbox):
        left, upper, right, lower = bbox
        cx, cy = K[idx, 0, 2], K[idx, 1, 2]

        new_cx = cx - left
        new_cy = cy - upper
        new_height = max(lower - upper, 1)
        new_width = max(right - left, 1)
        new_cx = new_width - new_cx
        new_cy = new_height - new_cy

        K[idx, 0, 2] = new_cx
        K[idx, 1, 2] = new_cy
        image_sizes.append((int(new_height), int(new_width)))

    return K, image_sizes


def perspective_projection(x3d, K, R=None, T=None):
    if R != None:
        x3d = torch.matmul(R, x3d.transpose(1, 2)).transpose(1, 2)
    if T != None:
        x3d = x3d + T.transpose(1, 2)

    x2d = torch.div(x3d, x3d[..., 2:])
    x2d = torch.matmul(K, x2d.transpose(-1, -2)).transpose(-1, -2)[..., :2]
    return x2d


def compute_bbox_from_points(X, img_w, img_h, scaleFactor=1.2):
    left = torch.clamp(X.min(1)[0][:, 0], min=0, max=img_w)
    right = torch.clamp(X.max(1)[0][:, 0], min=0, max=img_w)
    top = torch.clamp(X.min(1)[0][:, 1], min=0, max=img_h)
    bottom = torch.clamp(X.max(1)[0][:, 1], min=0, max=img_h)

    cx = (left + right) / 2
    cy = (top + bottom) / 2
    width = right - left
    height = bottom - top

    new_left = torch.clamp(cx - width / 2 * scaleFactor, min=0, max=img_w - 1)
    new_right = torch.clamp(cx + width / 2 * scaleFactor, min=1, max=img_w)
    new_top = torch.clamp(cy - height / 2 * scaleFactor, min=0, max=img_h - 1)
    new_bottom = torch.clamp(cy + height / 2 * scaleFactor, min=1, max=img_h)

    bbox = (
        torch.stack(
            (
                new_left.detach(),
                new_top.detach(),
                new_right.detach(),
                new_bottom.detach(),
            )
        )
        .int()
        .float()
        .T
    )
    return bbox


class Renderer:
    def __init__(self, width, height, K, device, faces=None):

        self.width = width
        self.height = height
        self.K = K

        self.device = device

        if faces is not None:
            self.faces = (
                torch.from_numpy((faces).astype("int")).unsqueeze(0).to(self.device)
            )

        self.initialize_camera_params()
        self.lights = PointLights(device=device, location=[[0.0, 0.0, -10.0]])
        self.create_renderer()

    def create_camera(self, R=None, T=None):
        if R is not None:
            self.R = R.clone().view(1, 3, 3).to(self.device)
        if T is not None:
            self.T = T.clone().view(1, 3).to(self.device)

        return PerspectiveCameras(
            device=self.device,
            R=self.R.mT,
            T=self.T,
            K=self.K_full,
            image_size=self.image_sizes,
            in_ndc=False,
        )

    def create_renderer(self):
        self.renderer = MeshRenderer(
            rasterizer=MeshRasterizer(
                raster_settings=RasterizationSettings(
                    image_size=self.image_sizes[0],
                    blur_radius=1e-5,
                ),
            ),
            shader=SoftPhongShader(
                device=self.device,
                lights=self.lights,
            ),
        )

    def create_normal_renderer(self):
        normal_renderer = MeshRenderer(
            rasterizer=MeshRasterizer(
                cameras=self.cameras,
                raster_settings=RasterizationSettings(
                    image_size=self.image_sizes[0],
                ),
            ),
            shader=NormalShader(device=self.device),
        )
        return normal_renderer

    def initialize_camera_params(self):
        """Hard coding for camera parameters
        TODO: Do some soft coding"""

        # Extrinsics
        self.R = (
            torch.diag(torch.tensor([1, 1, 1])).float().to(self.device).unsqueeze(0)
        )

        self.T = torch.tensor([0, 0, 0]).unsqueeze(0).float().to(self.device)

        # Intrinsics
        self.K = self.K.unsqueeze(0).float().to(self.device)
        self.bboxes = torch.tensor([[0, 0, self.width, self.height]]).float()
        self.K_full, self.image_sizes = update_intrinsics_from_bbox(self.K, self.bboxes)
        self.cameras = self.create_camera()

    def render_normal(self, vertices):
        vertices = vertices.unsqueeze(0)

        mesh = Meshes(verts=vertices, faces=self.faces)
        normal_renderer = self.create_normal_renderer()
        results = normal_renderer(mesh)
        results = torch.flip(results, [1, 2])
        return results

    def render_mesh(self, vertices, background, colors=[0.8, 0.8, 0.8]):

        self.update_bbox(vertices[::50], scale=1.2)
        vertices = vertices.unsqueeze(0)

        if colors[0] > 1:
            colors = [c / 255.0 for c in colors]
        verts_features = (
            torch.tensor(colors)
            .reshape(1, 1, 3)
            .to(device=vertices.device, dtype=vertices.dtype)
        )
        verts_features = verts_features.repeat(1, vertices.shape[1], 1)
        textures = TexturesVertex(verts_features=verts_features)

        mesh = Meshes(
            verts=vertices,
            faces=self.faces,
            textures=textures,
        )

        materials = Materials(device=self.device, specular_color=(colors,), shininess=0)

        results = torch.flip(
            self.renderer(
                mesh, materials=materials, cameras=self.cameras, lights=self.lights
            ),
            [1, 2],
        )
        image = results[0, ..., :3] * 255
        mask = results[0, ..., -1] > 1e-3

        image = overlay_image_onto_background(
            image, mask, self.bboxes, background.copy()
        )
        self.reset_bbox()
        return image

    def update_bbox(self, x3d, scale=2.0, mask=None):
        """Update bbox of cameras from the given 3d points

        x3d: input 3D keypoints (or vertices), (num_frames, num_points, 3)
        """
        if x3d.size(-1) != 3:
            x2d = x3d.unsqueeze(0)
        else:
            x2d = perspective_projection(
                x3d.unsqueeze(0), self.K, self.R, self.T.reshape(1, 3, 1)
            )

        if mask is not None:
            x2d = x2d[:, ~mask]
        bbox = compute_bbox_from_points(x2d, self.width, self.height, scale)
        self.bboxes = bbox

        self.K_full, self.image_sizes = update_intrinsics_from_bbox(self.K, bbox)
        self.cameras = self.create_camera()
        self.create_renderer()

    def reset_bbox(
        self,
    ):
        bbox = torch.zeros((1, 4)).float().to(self.device)
        bbox[0, 2] = self.width
        bbox[0, 3] = self.height
        self.bboxes = bbox

        self.K_full, self.image_sizes = update_intrinsics_from_bbox(self.K, bbox)
        self.cameras = self.create_camera()
        self.create_renderer()


class RendererUtil:
    def __init__(self, K, w, h, device, faces, keep_origin=True):
        self.keep_origin = keep_origin
        self.default_R = torch.eye(3)
        self.default_T = torch.zeros(3)
        self.device = device
        self.renderer = Renderer(w, h, K, device, faces)

    def set_extrinsic(self, R, T):
        self.default_R = R
        self.default_T = T

    def render_normal(self, verts_list):
        if not len(verts_list) == 1:
            return None

        self.renderer.create_camera(self.default_R, self.default_T)
        normal_map = self.renderer.render_normal(verts_list[0])
        return normal_map[0, :, :, 0]

    def render_frame(self, humans, pred_rend_array, verts_list=None, color_list=None):
        if not isinstance(pred_rend_array, np.ndarray):
            pred_rend_array = np.asarray(pred_rend_array)
        self.renderer.create_camera(self.default_R, self.default_T)
        _img = pred_rend_array
        if humans is not None:
            for human in humans:
                _img = self.renderer.render_mesh(human["v3d"].to(self.device), _img)
        else:
            for i, verts in enumerate(verts_list):
                if color_list is None:
                    _img = self.renderer.render_mesh(verts.to(self.device), _img)
                else:
                    _img = self.renderer.render_mesh(
                        verts.to(self.device), _img, color_list[i]
                    )
        if self.keep_origin:
            _img = np.concatenate([np.asarray(pred_rend_array), _img], 1).astype(
                np.uint8
            )
        return _img

    def render_video(self, results, pil_bis_frames, fps, out_path):
        writer = imageio.get_writer(
            out_path, fps=fps, mode="I", format="FFMPEG", macro_block_size=1
        )
        for i, humans in enumerate(tqdm(results)):
            pred_rend_array = pil_bis_frames[i]
            _img = self.render_frame(humans, pred_rend_array)
            try:
                writer.append_data(_img)
            except:
                print("Error in writing video")
                print(type(_img))
        writer.close()


def render_frame(
    renderer, humans, pred_rend_array, default_R, default_T, device, keep_origin=True
):

    if not isinstance(pred_rend_array, np.ndarray):
        pred_rend_array = np.asarray(pred_rend_array)
    renderer.create_camera(default_R, default_T)
    _img = pred_rend_array
    if humans is None:
        humans = []
    if isinstance(humans, dict):
        humans = [humans]
    for human in humans:
        if isinstance(human, dict):
            v3d = human["v3d"].to(device)
        else:
            v3d = human
        _img = renderer.render_mesh(v3d, _img)

    if keep_origin:
        _img = np.concatenate([np.asarray(pred_rend_array), _img], 1).astype(np.uint8)
    return _img


def render_video(
    results, faces, K, pil_bis_frames, fps, out_path, device, keep_origin=True
):
    # results [F, N, ...]
    if isinstance(pil_bis_frames[0], np.ndarray):
        height, width, _ = pil_bis_frames[0].shape
    else:
        shape = pil_bis_frames[0].size
        width, height = shape[1], shape[0]
    renderer = Renderer(width, height, K[0], device, faces)

    # build default camera
    default_R, default_T = torch.eye(3), torch.zeros(3)

    writer = imageio.get_writer(
        out_path, fps=fps, mode="I", format="FFMPEG", macro_block_size=1
    )
    for i, humans in enumerate(tqdm(results)):
        pred_rend_array = pil_bis_frames[i]
        _img = render_frame(
            renderer, humans, pred_rend_array, default_R, default_T, device, keep_origin
        )
        try:
            writer.append_data(_img)
        except:
            print("Error in writing video")
            print(type(_img))
    writer.close()