src/visualize/mano2mesh.py

from typing import Optional, Dict, Union
import numpy as np
import torch
from smplx import MANO
from smplx.utils import MANOOutput, Tensor
from smplx.lbs import lbs

from ..constant import MANO_MODEL_DIR


class ModifiedMANO(MANO):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def forward(
        self,
        betas: Optional[Tensor] = None,
        global_orient: Optional[Tensor] = None,
        hand_pose: Optional[Tensor] = None,
        transl: Optional[Tensor] = None,
    ) -> MANOOutput:
        '''
        betas: (B, 10)
        global_orient: (B, 3)
        hand_pose: (B, 45)
        transl: (B, 3)
        '''
        global_orient = global_orient if global_orient is not None else self.global_orient
        betas = betas if betas is not None else self.betas
        hand_pose = hand_pose if hand_pose is not None else self.hand_pose

        apply_trans = transl is not None or hasattr(self, 'transl')
        if transl is None:
            if hasattr(self, 'transl'):
                transl = self.transl

        full_pose = torch.cat([global_orient, hand_pose], dim=1)
        full_pose += self.pose_mean

        vertices, joints = lbs(
            betas, full_pose, self.v_template,
            self.shapedirs, self.posedirs,
            self.J_regressor, self.parents,
            self.lbs_weights, pose2rot=True
        )

        joints = self.vertex_joint_selector(vertices, joints)
        if self.joint_mapper is not None:
            joints = self.joint_mapper(joints)

        if apply_trans:
            joints = joints + transl.unsqueeze(dim=1).to(joints.device)
            vertices = vertices + transl.unsqueeze(dim=1).to(vertices.device)

        output = MANOOutput(
            vertices=vertices,
            joints=joints,
            betas=betas,
            global_orient=global_orient,
            hand_pose=hand_pose,
            full_pose=full_pose,
        )
        return output

left_manomodel = ModifiedMANO(
    model_path=MANO_MODEL_DIR,
    is_rhand=False,
    model_type='mano',
    use_pca=False,
    ext='pkl'
)

right_manomodel = ModifiedMANO(
    model_path=MANO_MODEL_DIR,
    is_rhand=True,
    model_type='mano',
    use_pca=False,
    ext='pkl'
)

def sequential_single_mano2mesh(
    hand:str,
    betas:np.ndarray | None,
    global_orient:np.ndarray | None,
    hand_pose:np.ndarray | None,
    transl:np.ndarray | None
):
    if hand == 'left':
        mano_model = left_manomodel
    else:
        mano_model = right_manomodel

    mano_model.to('cpu')

    output = mano_model(
        betas=torch.from_numpy(betas).float() if betas is not None else None,
        global_orient=torch.from_numpy(global_orient).float() if global_orient is not None else None,
        hand_pose=torch.from_numpy(hand_pose).float() if hand_pose is not None else None,
        transl=torch.from_numpy(transl).float() if transl is not None else None
    )

    vertices = output.vertices.clone().detach().cpu().numpy()
    faces = np.tile(mano_model.faces, (vertices.shape[0], 1, 1))

    return vertices, faces

def merge_two_meshes(
    left_vertices:np.ndarray,
    right_vertices:np.ndarray,
    left_faces:np.ndarray,
    right_faces:np.ndarray
) -> Dict[str, Union[np.ndarray, np.ndarray]]:
    '''
    Merge two meshes into one.
    left_vertices: (T, N, 3)
    right_vertices: (T, M, 3)
    left_faces: (T, F, 3)
    right_faces: (T, G, 3)
    '''
    vertices = np.concatenate([left_vertices, right_vertices], axis=1)
    faces = np.concatenate([
        left_faces,
        right_faces + left_vertices.shape[1]
    ], axis=1)

    return vertices, faces

def bihand_mano2mesh(
    left_motion:Dict[str, np.ndarray],
    right_motion:Dict[str, np.ndarray],
):
    '''
    motion:
        shape: (T, 10)
        pose: (T, 48)
        trans: (T, 3)
    '''
    T = left_motion['pose'].shape[0]
    left_vertices, left_faces = sequential_single_mano2mesh(
        'left',
        betas=left_motion['shape'] if 'shape' in left_motion else np.zeros((T, 10)),
        global_orient=left_motion['pose'][:, :3] if 'pose' in left_motion else np.zeros((T, 3)),
        hand_pose=left_motion['pose'][:, 3:] if 'pose' in left_motion else np.zeros((T, 45)),
        transl=left_motion['trans'] if 'trans' in left_motion else np.zeros((T, 3))
    )
    right_vertices, right_faces = sequential_single_mano2mesh(
        'right',
        betas=right_motion['shape'] if 'shape' in right_motion else np.zeros((T, 10)),
        global_orient=right_motion['pose'][:, :3] if 'pose' in right_motion else np.zeros((T, 3)),
        hand_pose=right_motion['pose'][:, 3:] if 'pose' in right_motion else np.zeros((T, 45)),
        transl=right_motion['trans'] if 'trans' in right_motion else np.zeros((T, 3))
    )
    vertices, faces = merge_two_meshes(
        left_vertices=left_vertices,
        right_vertices=right_vertices,
        left_faces=left_faces,
        right_faces=right_faces
    )
    return vertices, faces