hand_tracker.py

import cv2
import mediapipe as mp
import numpy as np


_MP_HANDS = getattr(getattr(mp, "solutions", None), "hands", None)
_HANDS = None
if _MP_HANDS is not None:
    _HANDS = _MP_HANDS.Hands(
        static_image_mode=False,
        max_num_hands=1,
        min_detection_confidence=0.5,
        min_tracking_confidence=0.5,
    )


def _empty_result(frame):
    return {
        "hand_found": False,
        "hand_center_x": None,
        "hand_center_y": None,
        "fingers_extended": {
            "thumb": False,
            "index": False,
            "middle": False,
            "ring": False,
            "pinky": False,
        },
        "landmarks": None,
        "frame": frame,
    }


def _finger_status(landmarks, handedness_label):
    # Landmark shortcuts for finger tips and lower joints.
    tips = {
        "thumb": _MP_HANDS.HandLandmark.THUMB_TIP,
        "index": _MP_HANDS.HandLandmark.INDEX_FINGER_TIP,
        "middle": _MP_HANDS.HandLandmark.MIDDLE_FINGER_TIP,
        "ring": _MP_HANDS.HandLandmark.RING_FINGER_TIP,
        "pinky": _MP_HANDS.HandLandmark.PINKY_TIP,
    }
    pips = {
        "thumb": _MP_HANDS.HandLandmark.THUMB_IP,
        "index": _MP_HANDS.HandLandmark.INDEX_FINGER_PIP,
        "middle": _MP_HANDS.HandLandmark.MIDDLE_FINGER_PIP,
        "ring": _MP_HANDS.HandLandmark.RING_FINGER_PIP,
        "pinky": _MP_HANDS.HandLandmark.PINKY_PIP,
    }

    status = {}
    for finger in ("index", "middle", "ring", "pinky"):
        tip_y = landmarks[tips[finger]].y
        pip_y = landmarks[pips[finger]].y
        status[finger] = tip_y < pip_y

    thumb_tip_x = landmarks[tips["thumb"]].x
    thumb_ip_x = landmarks[pips["thumb"]].x
    if handedness_label == "Right":
        status["thumb"] = thumb_tip_x < thumb_ip_x
    else:
        status["thumb"] = thumb_tip_x > thumb_ip_x

    return status


def detect_hand(frame):
    """
    Detect one hand from a webcam frame and return basic rule-friendly data.
    """
    if _HANDS is None:
        return _empty_result(frame)

    if frame is None:
        return _empty_result(frame)

    if not isinstance(frame, np.ndarray) or frame.size == 0:
        return _empty_result(frame)

    if frame.dtype != np.uint8:
        frame = np.clip(frame, 0, 255).astype(np.uint8)

    # Most Gradio webcam frames are RGB. Some environments may provide BGR.
    # Try RGB first, then fallback to BGR->RGB conversion.
    results = _HANDS.process(frame)
    if not results.multi_hand_landmarks and frame.ndim == 3 and frame.shape[2] == 3:
        rgb_fallback = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        results = _HANDS.process(rgb_fallback)

    if not results.multi_hand_landmarks:
        return _empty_result(frame)

    hand_landmarks = results.multi_hand_landmarks[0]
    handedness_label = "Right"
    if results.multi_handedness:
        handedness_label = results.multi_handedness[0].classification[0].label

    lm = hand_landmarks.landmark
    center_x = sum(point.x for point in lm) / len(lm)
    center_y = sum(point.y for point in lm) / len(lm)
    fingers = _finger_status(lm, handedness_label)

    return {
        "hand_found": True,
        "hand_center_x": center_x,
        "hand_center_y": center_y,
        "fingers_extended": fingers,
        "landmarks": hand_landmarks,
        "frame": frame,
    }