File size: 5,404 Bytes
5732928 5dff371 5732928 23dcf09 5732928 b49abeb 7a92420 5732928 7a92420 5732928 5dff371 9385963 7a92420 d52cb96 23dcf09 5732928 5b87ec4 5732928 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | import os
import cv2
import torch
import uuid
import trimesh
import numpy as np
import matplotlib.cm as cm
from mediapipe import solutions
from mediapipe.framework.formats import landmark_pb2
from lib.core.config import cfg
from lib.utils.human_models import mano
from lib.utils.mano_utils import change_flat_hand_mean
class ContactRenderer:
def __init__(self):
self.default_mesh_color = [130, 130, 130, 255]
self.contact_mesh_color = [0, 255, 0, 255]
# Initialize your hand model here
with torch.no_grad():
hand_pose = change_flat_hand_mean(np.zeros((48)), remove=True)
mano_rest_out = mano.layer['right'](betas=torch.zeros((1, 10)), hand_pose=torch.from_numpy(hand_pose[None, 3:]).float(), global_orient=torch.zeros((1, 3)), transl=torch.zeros((1, 3)))
self.hand_model_mano = trimesh.Trimesh(mano_rest_out.vertices[0], mano.watertight_face['right'])
def export_contact_mesh(self, contact_mask, output_path=None):
vis_contact = contact_mask == 1.0
self.hand_model_mano.visual.vertex_colors = np.tile(self.default_mesh_color, (self.hand_model_mano.vertices.shape[0], 1))
self.hand_model_mano.visual.vertex_colors[vis_contact] = self.contact_mesh_color
if output_path is None:
output_path = f"/tmp/contact_mesh_{uuid.uuid4().hex}.obj"
self.hand_model_mano.export(output_path)
return output_path
# This function is for demo code with mediapipe
MARGIN = 10 # pixels
FONT_SIZE = 1
FONT_THICKNESS = 1
HANDEDNESS_TEXT_COLOR = (88, 205, 54) # vibrant green
def draw_landmarks_on_image(rgb_image, detection_result):
hand_landmarks_list = detection_result.hand_landmarks
handedness_list = detection_result.handedness
annotated_image = np.copy(rgb_image)
right_hand_bbox = None
best_score = -1.0
best_idx = -1
# Step 1: Find the index of the most confident right hand
for idx in range(len(hand_landmarks_list)):
handedness = handedness_list[idx][0] # get Classification result
if handedness.category_name == "Right" and handedness.score > best_score:
best_score = handedness.score
best_idx = idx
# Step 2: If a right hand was found, draw it and extract bbox
if best_idx != -1:
hand_landmarks = hand_landmarks_list[best_idx]
handedness = handedness_list[best_idx]
# Convert landmarks to protobuf for drawing
hand_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
hand_landmarks_proto.landmark.extend([
landmark_pb2.NormalizedLandmark(x=lm.x, y=lm.y, z=lm.z) for lm in hand_landmarks
])
solutions.drawing_utils.draw_landmarks(
annotated_image,
hand_landmarks_proto,
solutions.hands.HAND_CONNECTIONS,
solutions.drawing_styles.get_default_hand_landmarks_style(),
solutions.drawing_styles.get_default_hand_connections_style())
# Compute bbox
height, width, _ = annotated_image.shape
x_coords = [lm.x * width for lm in hand_landmarks]
y_coords = [lm.y * height for lm in hand_landmarks]
x_min, x_max = int(min(x_coords)), int(max(x_coords))
y_min, y_max = int(min(y_coords)), int(max(y_coords))
bb_c_x, bb_c_y = (x_min+x_max)/2, (y_min+y_max)/2
bb_width, bb_height = x_max-x_min, y_max-y_min
expand_ratio = cfg.DATASET.ho_big_bbox_expand_ratio
bb_width_expand, bb_height_expand = expand_ratio * bb_width, expand_ratio * bb_height
x_min_expand, y_min_expand = bb_c_x - 0.5 * bb_width_expand, bb_c_y - 0.5 * bb_height_expand
right_hand_bbox = [x_min_expand, y_min_expand, bb_width_expand, bb_height_expand]
# Draw bbox and label
cv2.rectangle(annotated_image, (x_min, y_min), (x_max, y_max), (0, 255, 0), 2)
cv2.putText(annotated_image, "Right Hand", (x_min, y_min - MARGIN),
cv2.FONT_HERSHEY_DUPLEX, FONT_SIZE, HANDEDNESS_TEXT_COLOR, FONT_THICKNESS, cv2.LINE_AA)
return annotated_image, right_hand_bbox
def draw_landmarks_on_image_simple(rgb_image, right_hand_bbox):
# Draw bbox and label
annotated_image = rgb_image.copy()[..., ::-1]
annotated_image = np.ascontiguousarray(annotated_image)
# Expand bbox
x_min, x_max = int(right_hand_bbox[0]), int(right_hand_bbox[0]+right_hand_bbox[2])
y_min, y_max = int(right_hand_bbox[1]), int(right_hand_bbox[1]+right_hand_bbox[3])
bb_c_x, bb_c_y = (x_min+x_max)/2, (y_min+y_max)/2
bb_width, bb_height = x_max-x_min, y_max-y_min
expand_ratio = cfg.DATASET.ho_big_bbox_expand_ratio
bb_width_expand, bb_height_expand = expand_ratio * bb_width, expand_ratio * bb_height
x_min_expand, y_min_expand = bb_c_x - 0.5 * bb_width_expand, bb_c_y - 0.5 * bb_height_expand
right_hand_bbox = [x_min_expand, y_min_expand, bb_width_expand, bb_height_expand]
x_min, y_min, x_max, y_max = int(right_hand_bbox[0]), int(right_hand_bbox[1]), int(right_hand_bbox[0]+right_hand_bbox[2]), int(right_hand_bbox[1]+right_hand_bbox[3])
cv2.rectangle(annotated_image, (x_min, y_min), (x_max, y_max), (0, 255, 0), 2)
cv2.putText(annotated_image, "Right Hand", (x_min, y_min - MARGIN), cv2.FONT_HERSHEY_DUPLEX, FONT_SIZE, HANDEDNESS_TEXT_COLOR, FONT_THICKNESS, cv2.LINE_AA)
return annotated_image, right_hand_bbox |