# built-in dependencies from typing import Any, Dict, IO, List, Tuple, Union, Optional, cast from heapq import nlargest # 3rd part dependencies import numpy as np from numpy.typing import NDArray import cv2 # project dependencies from deepface.modules import modeling from deepface.models.Detector import Detector, DetectedFace, FacialAreaRegion from deepface.commons import image_utils from deepface.models.face_detection import OpenCv from deepface.modules.exceptions import FaceNotDetected from deepface.modules.exceptions import UnimplementedError, ImgNotFound from deepface.commons.logger import Logger logger = Logger() # pylint: disable=no-else-raise, too-many-positional-arguments def is_valid_landmark( coord: Optional[Union[Tuple[int, int], Tuple[float, float], List[Union[float, int]]]], width: int, height: int, ) -> bool: """ Check if a landmark coordinate is within valid image bounds. Args: coord (tuple or list or None): (x, y) coordinate to check. width (int): Image width. height (int): Image height. Returns: bool: True if coordinate is valid and within bounds, False otherwise. """ if coord is None: return False if not (isinstance(coord, (tuple, list)) and len(coord) == 2): return False x, y = coord return 0 <= x < width and 0 <= y < height def extract_faces( img_path: Union[str, NDArray[Any], IO[bytes], List[str], List[NDArray[Any]], List[IO[bytes]]], detector_backend: str = "opencv", enforce_detection: bool = True, align: bool = True, expand_percentage: int = 0, grayscale: bool = False, color_face: str = "rgb", normalize_face: bool = True, anti_spoofing: bool = False, max_faces: Optional[int] = None, ) -> Union[List[Dict[str, Any]], List[List[Dict[str, Any]]]]: """ Extract faces from a given image Args: img_path (str or list of str ornp.ndarray or IO[bytes]): Path to the first image. Accepts exact image path as a string, list of string, numpy array (BGR), a file object that supports at least `.read` and is opened in binary mode, or base64 encoded images. detector_backend (string): face detector backend. Options: 'opencv', 'retinaface', 'mtcnn', 'ssd', 'dlib', 'mediapipe', 'yolov8n', 'yolov8m', 'yolov8l', 'yolov11n', 'yolov11s', 'yolov11m', 'yolov11l', 'yolov12n', 'yolov12s', 'yolov12m','yolov12l' 'centerface' or 'skip' (default is opencv) enforce_detection (boolean): If no face is detected in an image, raise an exception. Default is True. Set to False to avoid the exception for low-resolution images. align (bool): Flag to enable face alignment (default is True). expand_percentage (int): expand detected facial area with a percentage. grayscale (boolean): (Deprecated) Flag to convert the output face image to grayscale (default is False). color_face (string): Color to return face image output. Options: 'rgb', 'bgr' or 'gray' (default is 'rgb'). normalize_face (boolean): Flag to enable normalization (divide by 255) of the output face image output face image normalization (default is True). anti_spoofing (boolean): Flag to enable anti spoofing (default is False). Returns: results (List[Dict[str, Any]]): A list of dictionaries, where each dictionary contains: - "face" (np.ndarray): The detected face as a NumPy array in RGB format. - "facial_area" (Dict[str, Any]): The detected face's regions as a dictionary containing: - keys 'x', 'y', 'w', 'h' with int values - keys 'left_eye', 'right_eye' with a tuple of 2 ints as values. left eye and right eye are eyes on the left and right respectively with respect to the person itself instead of observer. - "confidence" (float): The confidence score associated with the detected face. - "is_real" (boolean): antispoofing analyze result. this key is just available in the result only if anti_spoofing is set to True in input arguments. - "antispoof_score" (float): score of antispoofing analyze result. this key is just available in the result only if anti_spoofing is set to True in input arguments. """ if isinstance(img_path, list) or (isinstance(img_path, np.ndarray) and img_path.ndim == 4): if isinstance(img_path, np.ndarray): img_paths = [img_path[i] for i in range(img_path.shape[0])] else: img_paths = img_path all_faces: List[List[Dict[str, Any]]] = [] for single_img_path in img_paths: faces = cast( List[Dict[str, Any]], extract_faces( img_path=single_img_path, detector_backend=detector_backend, enforce_detection=enforce_detection, align=align, expand_percentage=expand_percentage, grayscale=grayscale, color_face=color_face, normalize_face=normalize_face, anti_spoofing=anti_spoofing, max_faces=max_faces, ), ) all_faces.append(faces) return all_faces resp_objs: List[Dict[str, Any]] = [] # img might be path, base64 or numpy array. Convert it to numpy whatever it is. img, img_name = image_utils.load_image(img_path) if img is None: raise ValueError(f"Exception while loading {img_name}") height, width, _ = img.shape base_region = FacialAreaRegion(x=0, y=0, w=width, h=height, confidence=0) if detector_backend == "skip": face_objs = [DetectedFace(img=img, facial_area=base_region, confidence=0)] else: face_objs = detect_faces( detector_backend=detector_backend, img=img, align=align, expand_percentage=expand_percentage, max_faces=max_faces, ) # in case of no face found if len(face_objs) == 0 and enforce_detection is True: if img_name is not None: raise FaceNotDetected( f"Face could not be detected in {img_name}." "Please confirm that the picture is a face photo " "or consider to set enforce_detection param to False." ) else: raise FaceNotDetected( "Face could not be detected. Please confirm that the picture is a face photo " "or consider to set enforce_detection param to False." ) if len(face_objs) == 0 and enforce_detection is False: face_objs = [DetectedFace(img=img, facial_area=base_region, confidence=0)] for face_obj in face_objs: current_img = face_obj.img current_region = face_obj.facial_area if current_img.shape[0] == 0 or current_img.shape[1] == 0: continue if grayscale is True: logger.warn("Parameter grayscale is deprecated. Use color_face instead.") current_img = cv2.cvtColor(current_img, cv2.COLOR_BGR2GRAY) else: if color_face == "rgb": current_img = current_img[:, :, ::-1] elif color_face == "bgr": pass # image is in BGR elif color_face == "gray": current_img = cv2.cvtColor(current_img, cv2.COLOR_BGR2GRAY) else: raise UnimplementedError( f"The color_face can be rgb, bgr or gray, but it is {color_face}." ) if normalize_face: current_img = current_img / 255 # normalize input in [0, 1] # cast to int for flask, and do final checks for borders x = max(0, int(current_region.x)) y = max(0, int(current_region.y)) w = min(width - x - 1, int(current_region.w)) h = min(height - y - 1, int(current_region.h)) landmarks = { "left_eye": current_region.left_eye, "right_eye": current_region.right_eye, "nose": current_region.nose, "mouth_left": current_region.mouth_left, "mouth_right": current_region.mouth_right, } # convert these to raw python types to ensure compability with Flask for key, value in landmarks.items(): if value is not None: landmarks[key] = (int(value[0]), int(value[1])) # Sanitize landmarks - set invalid ones to None for key, value in landmarks.items(): if not is_valid_landmark(value, width, height): landmarks[key] = None facial_area = { "x": x, "y": y, "w": w, "h": h, "left_eye": landmarks["left_eye"], "right_eye": landmarks["right_eye"], } # optional nose, mouth_left and mouth_right fields are coming just for retinaface if current_region.nose is not None: facial_area["nose"] = landmarks["nose"] if current_region.mouth_left is not None: facial_area["mouth_left"] = landmarks["mouth_left"] if current_region.mouth_right is not None: facial_area["mouth_right"] = landmarks["mouth_right"] resp_obj = { "face": current_img, "facial_area": facial_area, "confidence": round(float(current_region.confidence or 0), 2), } if anti_spoofing is True: antispoof_model = modeling.build_model(task="spoofing", model_name="Fasnet") is_real, antispoof_score = antispoof_model.analyze(img=img, facial_area=(x, y, w, h)) resp_obj["is_real"] = is_real resp_obj["antispoof_score"] = antispoof_score resp_objs.append(resp_obj) if len(resp_objs) == 0 and enforce_detection == True: raise ImgNotFound( f"Exception while extracting faces from {img_name}." "Consider to set enforce_detection arg to False." ) return resp_objs def detect_faces( detector_backend: str, img: NDArray[Any], align: bool = True, expand_percentage: int = 0, max_faces: Optional[int] = None, ) -> List[DetectedFace]: """ Detect face(s) from a given image Args: detector_backend (str): detector name img (np.ndarray): pre-loaded image align (bool): enable or disable alignment after detection expand_percentage (int): expand detected facial area with a percentage (default is 0). Returns: results (List[DetectedFace]): A list of DetectedFace objects where each object contains: - img (np.ndarray): The detected face as a NumPy array. - facial_area (FacialAreaRegion): The facial area region represented as x, y, w, h, left_eye and right eye. left eye and right eye are eyes on the left and right with respect to the person instead of observer. - confidence (float): The confidence score associated with the detected face. """ height, width, _ = img.shape face_detector: Detector = modeling.build_model( task="face_detector", model_name=detector_backend ) # validate expand percentage score if expand_percentage < 0: logger.warn( f"Expand percentage cannot be negative but you set it to {expand_percentage}." "Overwritten it to 0." ) expand_percentage = 0 # If faces are close to the upper boundary, alignment move them outside # Add a black border around an image to avoid this. height_border = int(0.5 * height) width_border = int(0.5 * width) if align is True: img = cv2.copyMakeBorder( img, height_border, height_border, width_border, width_border, cv2.BORDER_CONSTANT, value=[0, 0, 0], # Color of the border (black) ) # find facial areas of given image facial_areas = face_detector.detect_faces(img) if max_faces is not None and max_faces < len(facial_areas): facial_areas = nlargest( max_faces, facial_areas, key=lambda facial_area: facial_area.w * facial_area.h ) return [ extract_face( facial_area=facial_area, img=img, align=align, expand_percentage=expand_percentage, width_border=width_border, height_border=height_border, detector_backend=detector_backend, ) for facial_area in facial_areas ] def extract_face( facial_area: FacialAreaRegion, img: NDArray[Any], align: bool, expand_percentage: int, width_border: int, height_border: int, detector_backend: str, ) -> DetectedFace: x = facial_area.x y = facial_area.y w = facial_area.w h = facial_area.h left_eye = facial_area.left_eye right_eye = facial_area.right_eye confidence = facial_area.confidence nose = facial_area.nose mouth_left = facial_area.mouth_left mouth_right = facial_area.mouth_right if expand_percentage > 0: # Expand the facial region height and width by the provided percentage # ensuring that the expanded region stays within img.shape limits expanded_w = w + int(w * expand_percentage / 100) expanded_h = h + int(h * expand_percentage / 100) x = max(0, x - int((expanded_w - w) / 2)) y = max(0, y - int((expanded_h - h) / 2)) w = min(img.shape[1] - x, expanded_w) h = min(img.shape[0] - y, expanded_h) # extract detected face unaligned detected_face = img[int(y) : int(y + h), int(x) : int(x + w)] # use opencv if eyes aren't provided by the detector (e.g. ssd, yolo) if detector_backend != "opencv" and (left_eye is None or right_eye is None): default_detector: OpenCv.OpenCvClient = modeling.build_model( task="face_detector", model_name="opencv" ) left_eye_new, right_eye_new = default_detector.find_eyes(detected_face) if left_eye is None and left_eye_new is not None: left_eye = (int(left_eye_new[0]) + x, int(left_eye_new[1]) + y) logger.debug( f"left eye wasn't detected by {detector_backend}, overwritten by cv2 - {left_eye}" ) if right_eye is None and right_eye_new is not None: right_eye = (int(right_eye_new[0]) + x, int(right_eye_new[1]) + y) logger.debug( f"right eye wasn't detected by {detector_backend}, overwritten by cv2 - {right_eye}" ) # align original image, then find projection of detected face area after alignment if align is True: # and left_eye is not None and right_eye is not None: # we were aligning the original image before, but this comes with an extra cost # instead we now focus on the facial area with a margin # and align it instead of original image to decrese the cost sub_img, relative_x, relative_y = extract_sub_image(img=img, facial_area=(x, y, w, h)) aligned_sub_img, angle = align_img_wrt_eyes( img=sub_img, left_eye=left_eye, right_eye=right_eye ) rotated_x1, rotated_y1, rotated_x2, rotated_y2 = project_facial_area( facial_area=( relative_x, relative_y, relative_x + w, relative_y + h, ), angle=angle, size=(sub_img.shape[0], sub_img.shape[1]), ) detected_face = aligned_sub_img[ int(rotated_y1) : int(rotated_y2), int(rotated_x1) : int(rotated_x2) ] # do not spend memory for these temporary variables anymore del aligned_sub_img, sub_img # restore x, y, le and re before border added x = x - width_border y = y - height_border # w and h will not change if left_eye is not None: left_eye = (left_eye[0] - width_border, left_eye[1] - height_border) if right_eye is not None: right_eye = (right_eye[0] - width_border, right_eye[1] - height_border) if nose is not None: nose = (nose[0] - width_border, nose[1] - height_border) if mouth_left is not None: mouth_left = (mouth_left[0] - width_border, mouth_left[1] - height_border) if mouth_right is not None: mouth_right = (mouth_right[0] - width_border, mouth_right[1] - height_border) return DetectedFace( img=detected_face, facial_area=FacialAreaRegion( x=x, y=y, h=h, w=w, confidence=confidence, left_eye=left_eye, right_eye=right_eye, nose=nose, mouth_left=mouth_left, mouth_right=mouth_right, ), confidence=confidence or 0, ) def extract_sub_image( img: NDArray[Any], facial_area: Tuple[int, int, int, int] ) -> Tuple[NDArray[Any], int, int]: """ Get the sub image with given facial area while expanding the facial region to ensure alignment does not shift the face outside the image. This function doubles the height and width of the face region, and adds black pixels if necessary. Args: - img (np.ndarray): pre-loaded image with detected face - facial_area (tuple of int): Representing the (x, y, w, h) of the facial area. Returns: - extracted_face (np.ndarray): expanded facial image - relative_x (int): adjusted x-coordinates relative to the expanded region - relative_y (int): adjusted y-coordinates relative to the expanded region """ x, y, w, h = facial_area relative_x = int(0.5 * w) relative_y = int(0.5 * h) # calculate expanded coordinates x1, y1 = x - relative_x, y - relative_y x2, y2 = x + w + relative_x, y + h + relative_y # most of the time, the expanded region fits inside the image if x1 >= 0 and y1 >= 0 and x2 <= img.shape[1] and y2 <= img.shape[0]: return img[y1:y2, x1:x2], relative_x, relative_y # but sometimes, we need to add black pixels # ensure the coordinates are within bounds x1, y1 = max(0, x1), max(0, y1) x2, y2 = min(img.shape[1], x2), min(img.shape[0], y2) cropped_region = img[y1:y2, x1:x2] # create a black image extracted_face = np.zeros( (h + 2 * relative_y, w + 2 * relative_x, img.shape[2]), dtype=img.dtype ) # map the cropped region start_x = max(0, relative_x - x) start_y = max(0, relative_y - y) extracted_face[ start_y : start_y + cropped_region.shape[0], start_x : start_x + cropped_region.shape[1] ] = cropped_region return extracted_face, relative_x, relative_y def align_img_wrt_eyes( img: NDArray[Any], left_eye: Optional[Union[List[float], List[int], Tuple[float, float], Tuple[int, int]]], right_eye: Optional[Union[List[float], List[int], Tuple[float, float], Tuple[int, int]]], ) -> Tuple[NDArray[Any], float]: """ Align a given image horizantally with respect to their left and right eye locations Args: img (np.ndarray): pre-loaded image with detected face left_eye (list or tuple): coordinates of left eye with respect to the person itself right_eye(list or tuple): coordinates of right eye with respect to the person itself Returns: img (np.ndarray): aligned facial image """ # if eye could not be detected for the given image, return image itself if left_eye is None or right_eye is None: return img, 0 # sometimes unexpectedly detected images come with nil dimensions if img.shape[0] == 0 or img.shape[1] == 0: return img, 0 angle = float(np.degrees(np.arctan2(left_eye[1] - right_eye[1], left_eye[0] - right_eye[0]))) (h, w) = img.shape[:2] center = (w // 2, h // 2) M = cv2.getRotationMatrix2D(center, angle, 1.0) img = cv2.warpAffine( img, M, (w, h), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0) ) return img, angle def project_facial_area( facial_area: Tuple[int, int, int, int], angle: float, size: Tuple[int, int] ) -> Tuple[int, int, int, int]: """ Update pre-calculated facial area coordinates after image itself rotated with respect to the eyes. Inspried from the work of @UmutDeniz26 - github.com/serengil/retinaface/pull/80 Args: facial_area (tuple of int): Representing the (x1, y1, x2, y2) of the facial area. x2 is equal to x1 + w1, and y2 is equal to y1 + h1 angle (float): Angle of rotation in degrees. Its sign determines the direction of rotation. Note that angles > 360 degrees are normalized to the range [0, 360). size (tuple of int): Tuple representing the size of the image (width, height). Returns: rotated_coordinates (tuple of int): Representing the new coordinates (x1, y1, x2, y2) or (x1, y1, x1+w1, y1+h1) of the rotated facial area. """ # Normalize the witdh of the angle so we don't have to # worry about rotations greater than 360 degrees. # We workaround the quirky behavior of the modulo operator # for negative angle values. direction = 1 if angle >= 0 else -1 angle = abs(angle) % 360 if angle == 0: return facial_area # Angle in radians angle = angle * np.pi / 180 height, weight = size # Translate the facial area to the center of the image x = (facial_area[0] + facial_area[2]) / 2 - weight / 2 y = (facial_area[1] + facial_area[3]) / 2 - height / 2 # Rotate the facial area x_new = x * np.cos(angle) + y * direction * np.sin(angle) y_new = -x * direction * np.sin(angle) + y * np.cos(angle) # Translate the facial area back to the original position x_new = x_new + weight / 2 y_new = y_new + height / 2 # Calculate projected coordinates after alignment x1 = x_new - (facial_area[2] - facial_area[0]) / 2 y1 = y_new - (facial_area[3] - facial_area[1]) / 2 x2 = x_new + (facial_area[2] - facial_area[0]) / 2 y2 = y_new + (facial_area[3] - facial_area[1]) / 2 # validate projected coordinates are in image's boundaries x1 = max(int(x1), 0) y1 = max(int(y1), 0) x2 = min(int(x2), weight) y2 = min(int(y2), height) return (x1, y1, x2, y2)