# Multi-HMR # Copyright (c) 2024-present NAVER Corp. # CC BY-NC-SA 4.0 license import numpy as np import torch.nn.functional as F import torch import roma from smplx.joint_names import JOINT_NAMES def rot6d_to_rotmat(x): """ 6D rotation representation to 3x3 rotation matrix. Args: x: (B,6) Batch of 6-D rotation representations. Returns: torch.Tensor: Batch of corresponding rotation matrices with shape (B,3,3). """ x = x.reshape(-1, 2, 3).permute(0, 2, 1).contiguous() y = roma.special_gramschmidt(x) return y def get_smplx_joint_names(*args, **kwargs): return JOINT_NAMES[:127] COCO17_JOINTS_NAME = { 0: "nose", 1: "left_eye", 2: "right_eye", 3: "left_ear", 4: "right_ear", 5: "left_shoulder", 6: "right_shoulder", 7: "left_elbow", 8: "right_elbow", 9: "left_wrist", 10: "right_wrist", 11: "left_hip", 12: "right_hip", 13: "left_knee", 14: "right_knee", 15: "left_ankle", 16: "right_ankle", } OPENPOSE25_JOINTS_NAME = { 0: "nose", 1: "neck", 2: "right_shoulder", 3: "right_elbow", 4: "right_wrist", 5: "left_shoulder", 6: "left_elbow", 7: "left_wrist", 8: "MidHip", 9: "right_hip", 10: "right_knee", 11: "right_ankle", 12: "left_hip", 13: "left_knee", 14: "left_ankle", 15: "right_eye", 16: "left_eye", 17: "right_ear", 18: "left_ear", 19: "LBigToe", 20: "LSmallToe", 21: "left_heel", 22: "RBigToe", 23: "RSmallToe", 24: "right_heel", } def joints_smplx_to_coco(): smplx_joints_name = get_smplx_joint_names() joints_idx = [] for k, v in COCO17_JOINTS_NAME.items(): joints_idx.append(smplx_joints_name.index(v)) return joints_idx def joints_openpose25_to_coco17(): idx_list = [0] * 17 is_found = False for coco_key, coco_value in COCO17_JOINTS_NAME.items(): is_found = False for openpose_key, openpose_value in OPENPOSE25_JOINTS_NAME.items(): if coco_value == openpose_value: idx_list[coco_key] = openpose_key is_found = True break assert is_found, f"{coco_key} is not found in openpose keypoints" return idx_list COCO_WHOLEBODY_KEYPOINTS = [ "nose", "left_eye", "right_eye", "left_ear", "right_ear", "left_shoulder", "right_shoulder", "left_elbow", "right_elbow", "left_wrist", "right_wrist", "left_hip", "right_hip", "left_knee", "right_knee", "left_ankle", "right_ankle", "left_bigtoe", "left_smalltoe", "left_heel", "right_bigtoe", "right_smalltoe", "right_heel", "right_contour_1", # original name: face_contour_1 "right_contour_2", # original name: face_contour_2 "right_contour_3", # original name: face_contour_3 "right_contour_4", # original name: face_contour_4 "right_contour_5", # original name: face_contour_5 "right_contour_6", # original name: face_contour_6 "right_contour_7", # original name: face_contour_7 "right_contour_8", # original name: face_contour_8 "contour_middle", # original name: face_contour_9 "left_contour_8", # original name: face_contour_10 "left_contour_7", # original name: face_contour_11 "left_contour_6", # original name: face_contour_12 "left_contour_5", # original name: face_contour_13 "left_contour_4", # original name: face_contour_14 "left_contour_3", # original name: face_contour_15 "left_contour_2", # original name: face_contour_16 "left_contour_1", # original name: face_contour_17 "right_eyebrow_1", "right_eyebrow_2", "right_eyebrow_3", "right_eyebrow_4", "right_eyebrow_5", "left_eyebrow_5", "left_eyebrow_4", "left_eyebrow_3", "left_eyebrow_2", "left_eyebrow_1", "nosebridge_1", "nosebridge_2", "nosebridge_3", "nosebridge_4", "right_nose_2", # original name: nose_1 "right_nose_1", # original name: nose_2 "nose_middle", # original name: nose_3 "left_nose_1", # original name: nose_4 "left_nose_2", # original name: nose_5 "right_eye_1", "right_eye_2", "right_eye_3", "right_eye_4", "right_eye_5", "right_eye_6", "left_eye_4", "left_eye_3", "left_eye_2", "left_eye_1", "left_eye_6", "left_eye_5", "right_mouth_1", # original name: mouth_1 "right_mouth_2", # original name: mouth_2 "right_mouth_3", # original name: mouth_3 "mouth_top", # original name: mouth_4 "left_mouth_3", # original name: mouth_5 "left_mouth_2", # original name: mouth_6 "left_mouth_1", # original name: mouth_7 "left_mouth_5", # original name: mouth_8 "left_mouth_4", # original name: mouth_9 "mouth_bottom", # original name: mouth_10 "right_mouth_4", # original name: mouth_11 "right_mouth_5", # original name: mouth_12 "right_lip_1", # original name: lip_1 "right_lip_2", # original name: lip_2 "lip_top", # original name: lip_3 "left_lip_2", # original name: lip_4 "left_lip_1", # original name: lip_5 "left_lip_3", # original name: lip_6 "lip_bottom", # original name: lip_7 "right_lip_3", # original name: lip_8 "left_hand_root", "left_thumb_1", "left_thumb_2", "left_thumb_3", "left_thumb", "left_index_1", "left_index_2", "left_index_3", "left_index", "left_middle_1", "left_middle_2", "left_middle_3", "left_middle", "left_ring_1", "left_ring_2", "left_ring_3", "left_ring", "left_pinky_1", "left_pinky_2", "left_pinky_3", "left_pinky", "right_hand_root", "right_thumb_1", "right_thumb_2", "right_thumb_3", "right_thumb", "right_index_1", "right_index_2", "right_index_3", "right_index", "right_middle_1", "right_middle_2", "right_middle_3", "right_middle", "right_ring_1", "right_ring_2", "right_ring_3", "right_ring", "right_pinky_1", "right_pinky_2", "right_pinky_3", "right_pinky", ] SMPLX_KEYPOINTS = [ "pelvis", "left_hip", "right_hip", "spine_1", "left_knee", "right_knee", "spine_2", "left_ankle", "right_ankle", "spine_3", "left_foot", "right_foot", "neck", "left_collar", "right_collar", "head", "left_shoulder", "right_shoulder", "left_elbow", "right_elbow", "left_wrist", "right_wrist", "jaw", "left_eyeball", "right_eyeball", "left_index_1", "left_index_2", "left_index_3", "left_middle_1", "left_middle_2", "left_middle_3", "left_pinky_1", "left_pinky_2", "left_pinky_3", "left_ring_1", "left_ring_2", "left_ring_3", "left_thumb_1", "left_thumb_2", "left_thumb_3", "right_index_1", "right_index_2", "right_index_3", "right_middle_1", "right_middle_2", "right_middle_3", "right_pinky_1", "right_pinky_2", "right_pinky_3", "right_ring_1", "right_ring_2", "right_ring_3", "right_thumb_1", "right_thumb_2", "right_thumb_3", "nose", "right_eye", "left_eye", "right_ear", "left_ear", "left_bigtoe", "left_smalltoe", "left_heel", "right_bigtoe", "right_smalltoe", "right_heel", "left_thumb", "left_index", "left_middle", "left_ring", "left_pinky", "right_thumb", "right_index", "right_middle", "right_ring", "right_pinky", "right_eyebrow_1", "right_eyebrow_2", "right_eyebrow_3", "right_eyebrow_4", "right_eyebrow_5", "left_eyebrow_5", "left_eyebrow_4", "left_eyebrow_3", "left_eyebrow_2", "left_eyebrow_1", "nosebridge_1", "nosebridge_2", "nosebridge_3", "nosebridge_4", "right_nose_2", # original name: nose_1 "right_nose_1", # original name: nose_2 "nose_middle", # original name: nose_3 "left_nose_1", # original name: nose_4 "left_nose_2", # original name: nose_5 "right_eye_1", "right_eye_2", "right_eye_3", "right_eye_4", "right_eye_5", "right_eye_6", "left_eye_4", "left_eye_3", "left_eye_2", "left_eye_1", "left_eye_6", "left_eye_5", "right_mouth_1", # original name: mouth_1 "right_mouth_2", # original name: mouth_2 "right_mouth_3", # original name: mouth_3 "mouth_top", # original name: mouth_4 "left_mouth_3", # original name: mouth_5 "left_mouth_2", # original name: mouth_6 "left_mouth_1", # original name: mouth_7 "left_mouth_5", # original name: mouth_8 "left_mouth_4", # original name: mouth_9 "mouth_bottom", # original name: mouth_10 "right_mouth_4", # original name: mouth_11 "right_mouth_5", # original name: mouth_12 "right_lip_1", # original name: lip_1 "right_lip_2", # original name: lip_2 "lip_top", # original name: lip_3 "left_lip_2", # original name: lip_4 "left_lip_1", # original name: lip_5 "left_lip_3", # original name: lip_6 "lip_bottom", # original name: lip_7 "right_lip_3", # original name: lip_8 "right_contour_1", # original name: face_contour_1 "right_contour_2", # original name: face_contour_2 "right_contour_3", # original name: face_contour_3 "right_contour_4", # original name: face_contour_4 "right_contour_5", # original name: face_contour_5 "right_contour_6", # original name: face_contour_6 "right_contour_7", # original name: face_contour_7 "right_contour_8", # original name: face_contour_8 "contour_middle", # original name: face_contour_9 "left_contour_8", # original name: face_contour_10 "left_contour_7", # original name: face_contour_11 "left_contour_6", # original name: face_contour_12 "left_contour_5", # original name: face_contour_13 "left_contour_4", # original name: face_contour_14 "left_contour_3", # original name: face_contour_15 "left_contour_2", # original name: face_contour_16 "left_contour_1", # original name: face_contour_17 ] LEFT_HAND_KEYPOINTS = [ "left_wrist", "left_index_1", "left_index_2", "left_index_3", "left_middle_1", "left_middle_2", "left_middle_3", "left_pinky_1", "left_pinky_2", "left_pinky_3", "left_ring_1", "left_ring_2", "left_ring_3", "left_thumb_1", "left_thumb_2", "left_thumb_3", ] RIGHT_HAND_KEYPOINTS = [ "right_wrist", "right_index_1", "right_index_2", "right_index_3", "right_middle_1", "right_middle_2", "right_middle_3", "right_pinky_1", "right_pinky_2", "right_pinky_3", "right_ring_1", "right_ring_2", "right_ring_3", "right_thumb_1", "right_thumb_2", "right_thumb_3", ] COCO_PLUS_KEYPOINTS = [ "nose", "left_eye", "right_eye", "left_ear", "right_ear", "left_shoulder", "right_shoulder", "left_elbow", "right_elbow", "left_wrist", "right_wrist", "left_hip", "right_hip", "left_knee", "right_knee", "left_ankle", "right_ankle", "left_bigtoe", "left_smalltoe", "left_heel", "right_bigtoe", "right_smalltoe", "right_heel", ] KEYPOINTS_FACTORY = { "smplx": SMPLX_KEYPOINTS, "coco_wholebody": COCO_WHOLEBODY_KEYPOINTS, "left_hand": LEFT_HAND_KEYPOINTS, "right_hand": RIGHT_HAND_KEYPOINTS, "coco_plus": COCO_PLUS_KEYPOINTS, } MAPPING_CACHE = {} def get_mapping( src: str, dst: str, keypoints_factory: dict = KEYPOINTS_FACTORY, ): """Get mapping list from src to dst. Args: src (str): source data type from keypoints_factory. dst (str): destination data type from keypoints_factory. approximate (bool): control whether approximate mapping is allowed. keypoints_factory (dict, optional): A class to store the attributes. Defaults to keypoints_factory. Returns: list: [src_to_intersection_idx, dst_to_intersection_index, intersection_names] """ if ( src.lower() in MAPPING_CACHE.keys() and dst.lower() in MAPPING_CACHE[src.lower()].keys() ): return MAPPING_CACHE[src.lower()][dst.lower()] src_names = keypoints_factory[src.lower()] dst_names = keypoints_factory[dst.lower()] dst_idxs, src_idxs, intersection = [], [], [] full_mapping_idx = [] unmapped_names, approximate_names = [], [] for dst_idx, dst_name in enumerate(dst_names): try: src_idx = src_names.index(dst_name) except ValueError: src_idx = -1 if src_idx >= 0: dst_idxs.append(dst_idx) src_idxs.append(src_idx) intersection.append(dst_name) full_mapping_idx.append(src_idx) # approximate mapping mapping_list = (dst_idxs, src_idxs, intersection, full_mapping_idx) if not src.lower() in MAPPING_CACHE.keys(): MAPPING_CACHE[src.lower()] = {} MAPPING_CACHE[src.lower()][dst.lower()] = mapping_list return mapping_list if __name__ == "__main__": print(joints_smplx_to_coco())