Spaces:
Sleeping
Sleeping
File size: 4,695 Bytes
40ac571 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 |
import os
import cv2
import numpy as np
from .mp_utils import LMKExtractor
from .draw_util import FaceMeshVisualizer
from .pose_util import project_points_with_trans, matrix_to_euler_and_translation, euler_and_translation_to_matrix
class FaceMeshDetector:
"""
Class for face mesh detection and landmark extraction.
"""
def __init__(self) -> None:
self.lmk_extractor = LMKExtractor()
self.vis = FaceMeshVisualizer(forehead_edge=False, iris_edge=False, iris_point=True)
def __call__(self, image: np.array):
frame_bgr = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
try:
face_result = self.lmk_extractor(frame_bgr)
face_result['width'] = frame_bgr.shape[1]
face_result['height'] = frame_bgr.shape[0]
except:
face_result = None
if face_result is None:
return np.zeros_like(frame_bgr), None
lmks = face_result['lmks'].astype(np.float32)
motion = self.vis.draw_landmarks((frame_bgr.shape[1], frame_bgr.shape[0]), lmks, normed=True)
return motion, face_result
def smooth_pose_seq(pose_seq, window_size=5):
smoothed_pose_seq = np.zeros_like(pose_seq)
for i in range(len(pose_seq)):
start = max(0, i - window_size // 2)
end = min(len(pose_seq), i + window_size // 2 + 1)
smoothed_pose_seq[i] = np.mean(pose_seq[start:end], axis=0)
return smoothed_pose_seq
class FaceMeshAlign():
def __init__(self):
self.vis = FaceMeshVisualizer(forehead_edge=False, iris_edge=False, iris_point=True)
def _scale_iris(self, lmks, verts):
r_iris_ids = [468, 469, 470, 471, 472]
l_iris_ids = [473, 474, 475, 476, 477]
r_eye_ids = [33,7,163,144,145,153,154,155,246,161,160,159,158,157,173,133]
l_eye_ids = [249,390,373,374,380,381,382,362,466,388,387,386,385,384,398,263]
def scale_iris(iris_ids, eye_ids):
iris_lmks, eye_lmks, eye_verts = lmks[iris_ids], lmks[eye_ids], verts[eye_ids]
x0, y0, x1, y1 = np.min(eye_lmks[:, 0]), np.min(eye_lmks[:, 1]), np.max(eye_lmks[:, 0]), np.max(eye_lmks[:, 1])
iris_lmks[:, 0] = (iris_lmks[:, 0] - x0) / (x1 - x0)
iris_lmks[:, 1] = (iris_lmks[:, 1] - y0) / (y1 - y0)
iris_verts = np.zeros((5, 2))
x0, y0, x1, y1 = np.min(eye_verts[:, 0]), np.min(eye_verts[:, 1]), np.max(eye_verts[:, 0]), np.max(eye_verts[:, 1])
iris_verts[:, 0] = iris_lmks[:, 0] * (x1 - x0) + x0
iris_verts[:, 1] = iris_lmks[:, 1] * (y1 - y0) + y0
return iris_verts
r_iris_verts = scale_iris(r_iris_ids, r_eye_ids)
l_iris_verts = scale_iris(l_iris_ids, l_eye_ids)
verts = np.vstack((verts, r_iris_verts, l_iris_verts))
return verts
def __call__(self, ref_result, temp_results):
width, height = ref_result['width'], ref_result['height']
# prepare template data
trans_mat_arr = np.array([x['trans_mat'] for x in temp_results])
verts_arr = np.array([x['lmks3d'] for x in temp_results])
bs_arr = np.array([x['bs'] for x in temp_results])
min_bs_idx = np.argmin(bs_arr.sum(1))
# compute delta pose
pose_arr = np.zeros([trans_mat_arr.shape[0], 6])
for i in range(pose_arr.shape[0]):
euler_angles, translation_vector = matrix_to_euler_and_translation(trans_mat_arr[i]) # real pose of source
pose_arr[i, :3] = euler_angles
pose_arr[i, 3:6] = translation_vector
init_tran_vec = ref_result['trans_mat'][:3, 3] # init translation of tgt
pose_arr[:, 3:6] = pose_arr[:, 3:6] - pose_arr[0, 3:6] + init_tran_vec # (relative translation of source) + (init translation of tgt)
pose_arr_smooth = smooth_pose_seq(pose_arr, window_size=1)
pose_mat_smooth = [euler_and_translation_to_matrix(pose_arr_smooth[i][:3], pose_arr_smooth[i][3:6]) for i in range(pose_arr_smooth.shape[0])]
pose_mat_smooth = np.array(pose_mat_smooth)
# face retarget
verts_arr = verts_arr - verts_arr[min_bs_idx] + ref_result['lmks3d']
# project 3D mesh to 2D landmark
projected_vertices = project_points_with_trans(verts_arr, pose_mat_smooth, [height, width])
pose_list = []
for i, verts in enumerate(projected_vertices):
verts = self._scale_iris(temp_results[i]['lmks'], verts)
lmk_img = self.vis.draw_landmarks((width, height), verts, normed=False)
pose_list.append(lmk_img)
pose_list = np.array(pose_list)[1:]
return pose_list |