lib/utils/vis_utils.py

import os
import cv2
import torch
import uuid
import trimesh
import numpy as np
import matplotlib.cm as cm

from mediapipe import solutions
from mediapipe.framework.formats import landmark_pb2

from lib.core.config import cfg
from lib.utils.human_models import mano
from lib.utils.mano_utils import change_flat_hand_mean



class ContactRenderer:
    def __init__(self):
        self.default_mesh_color = [130, 130, 130, 255]
        self.contact_mesh_color = [0, 255, 0, 255]
        # Initialize your hand model here
        with torch.no_grad():
            hand_pose = change_flat_hand_mean(np.zeros((48)), remove=True)
            mano_rest_out = mano.layer['right'](betas=torch.zeros((1, 10)), hand_pose=torch.from_numpy(hand_pose[None, 3:]).float(), global_orient=torch.zeros((1, 3)), transl=torch.zeros((1, 3)))
            self.hand_model_mano = trimesh.Trimesh(mano_rest_out.vertices[0], mano.watertight_face['right'])

    def export_contact_mesh(self, contact_mask, output_path=None):
        vis_contact = contact_mask == 1.0
        self.hand_model_mano.visual.vertex_colors = np.tile(self.default_mesh_color, (self.hand_model_mano.vertices.shape[0], 1))
        self.hand_model_mano.visual.vertex_colors[vis_contact] = self.contact_mesh_color
        if output_path is None:
            output_path = f"/tmp/contact_mesh_{uuid.uuid4().hex}.obj"
        self.hand_model_mano.export(output_path)
        return output_path


# This function is for demo code with mediapipe
MARGIN = 10  # pixels
FONT_SIZE = 1
FONT_THICKNESS = 1
HANDEDNESS_TEXT_COLOR = (88, 205, 54) # vibrant green


def draw_landmarks_on_image(rgb_image, detection_result):
    hand_landmarks_list = detection_result.hand_landmarks
    handedness_list = detection_result.handedness
    annotated_image = np.copy(rgb_image)
    right_hand_bbox = None

    best_score = -1.0
    best_idx = -1

    # Step 1: Find the index of the most confident right hand
    for idx in range(len(hand_landmarks_list)):
        handedness = handedness_list[idx][0]  # get Classification result
        if handedness.category_name == "Right" and handedness.score > best_score:
            best_score = handedness.score
            best_idx = idx

    # Step 2: If a right hand was found, draw it and extract bbox
    if best_idx != -1:
        hand_landmarks = hand_landmarks_list[best_idx]
        handedness = handedness_list[best_idx]

        # Convert landmarks to protobuf for drawing
        hand_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
        hand_landmarks_proto.landmark.extend([
            landmark_pb2.NormalizedLandmark(x=lm.x, y=lm.y, z=lm.z) for lm in hand_landmarks
        ])
        solutions.drawing_utils.draw_landmarks(
            annotated_image,
            hand_landmarks_proto,
            solutions.hands.HAND_CONNECTIONS,
            solutions.drawing_styles.get_default_hand_landmarks_style(),
            solutions.drawing_styles.get_default_hand_connections_style())

        # Compute bbox
        height, width, _ = annotated_image.shape
        x_coords = [lm.x * width for lm in hand_landmarks]
        y_coords = [lm.y * height for lm in hand_landmarks]
        x_min, x_max = int(min(x_coords)), int(max(x_coords))
        y_min, y_max = int(min(y_coords)), int(max(y_coords))
        bb_c_x, bb_c_y = (x_min+x_max)/2, (y_min+y_max)/2
        bb_width, bb_height = x_max-x_min, y_max-y_min

        expand_ratio = cfg.DATASET.ho_big_bbox_expand_ratio

        bb_width_expand, bb_height_expand = expand_ratio * bb_width, expand_ratio * bb_height
        x_min_expand, y_min_expand = bb_c_x - 0.5 * bb_width_expand, bb_c_y - 0.5 * bb_height_expand
        
        right_hand_bbox = [x_min_expand, y_min_expand, bb_width_expand, bb_height_expand]

        # Draw bbox and label
        cv2.rectangle(annotated_image, (x_min, y_min), (x_max, y_max), (0, 255, 0), 2)
        cv2.putText(annotated_image, "Right Hand", (x_min, y_min - MARGIN),
                    cv2.FONT_HERSHEY_DUPLEX, FONT_SIZE, HANDEDNESS_TEXT_COLOR, FONT_THICKNESS, cv2.LINE_AA)

    return annotated_image, right_hand_bbox


def draw_landmarks_on_image_simple(rgb_image, right_hand_bbox):
    # Draw bbox and label
    annotated_image = rgb_image.copy()[..., ::-1]
    annotated_image = np.ascontiguousarray(annotated_image)

    # Expand bbox
    x_min, x_max = int(right_hand_bbox[0]), int(right_hand_bbox[0]+right_hand_bbox[2])
    y_min, y_max = int(right_hand_bbox[1]), int(right_hand_bbox[1]+right_hand_bbox[3])
    bb_c_x, bb_c_y = (x_min+x_max)/2, (y_min+y_max)/2
    bb_width, bb_height = x_max-x_min, y_max-y_min

    expand_ratio = cfg.DATASET.ho_big_bbox_expand_ratio

    bb_width_expand, bb_height_expand = expand_ratio * bb_width, expand_ratio * bb_height
    x_min_expand, y_min_expand = bb_c_x - 0.5 * bb_width_expand, bb_c_y - 0.5 * bb_height_expand
    
    right_hand_bbox = [x_min_expand, y_min_expand, bb_width_expand, bb_height_expand]

    x_min, y_min, x_max, y_max = int(right_hand_bbox[0]), int(right_hand_bbox[1]), int(right_hand_bbox[0]+right_hand_bbox[2]), int(right_hand_bbox[1]+right_hand_bbox[3])
    cv2.rectangle(annotated_image, (x_min, y_min), (x_max, y_max), (0, 255, 0), 2)
    cv2.putText(annotated_image, "Right Hand", (x_min, y_min - MARGIN), cv2.FONT_HERSHEY_DUPLEX, FONT_SIZE, HANDEDNESS_TEXT_COLOR, FONT_THICKNESS, cv2.LINE_AA)

    return annotated_image, right_hand_bbox