app.py

import gradio as gr
import cv2
import torch
import numpy as np
from PIL import Image
import os
import sys
from pathlib import Path

# Add src to path
sys.path.append(str(Path(__file__).parent))

from src.models.student import LIPEV2StudentGaze360Gold
from src.utils.preprocess import GazePreprocessor

# 1. Model Initialization
device = "cuda" if torch.cuda.is_available() else "cpu"
model = LIPEV2StudentGaze360Gold()

# Load the best available checkpoint
checkpoint_path = "checkpoints/swa_gold_p11.pt"
if not os.path.exists(checkpoint_path):
    # Fallback search
    pts = list(Path("checkpoints").rglob("*.pt"))
    if pts:
        checkpoint_path = str(pts[0])

if os.path.exists(checkpoint_path):
    checkpoint = torch.load(checkpoint_path, map_location=device)
    model.load_state_dict(checkpoint)
    print(f"Loaded {checkpoint_path}")

model.to(device)
model.eval()

# 2. Preprocessor Initialization
# Note: face_landmarker.task must be in the root directory for HF Spaces
preprocessor = GazePreprocessor(model_path='face_landmarker.task')

def draw_gaze(frame, pitch, yaw, landmarks):
    h, w, _ = frame.shape
    # Simple visualization logic
    # Calculate eye centers (using indices from preprocessor)
    left_eye_center = np.mean([[landmarks[i].x * w, landmarks[i].y * h] for i in [362, 263]], axis=0).astype(int)
    right_eye_center = np.mean([[landmarks[i].x * w, landmarks[i].y * h] for i in [33, 133]], axis=0).astype(int)
    
    p_rad = np.radians(pitch)
    y_rad = np.radians(yaw)
    
    dx = -np.sin(y_rad) * np.cos(p_rad)
    dy = -np.sin(p_rad)
    
    length = 80
    for center in [left_eye_center, right_eye_center]:
        end_point = (int(center[0] + dx * length), int(center[1] + dy * length))
        cv2.arrowedLine(frame, tuple(center), end_point, (0, 255, 0), 2, tipLength=0.3)
    return frame

def predict(img):
    if img is None:
        return None, "Please upload an image."
    
    # Convert PIL to BGR
    frame = np.array(img)
    frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
    
    # Get landmarks
    landmarks = preprocessor.get_landmarks(frame)
    if not landmarks:
        return img, "No face detected."
    
    # Prepare landmarks (zero-centered)
    landmarks_arr = np.array([[lm.x, lm.y] for lm in landmarks])
    left_c = np.mean([[landmarks[idx].x, landmarks[idx].y] for idx in [362, 263]], axis=0)
    right_c = np.mean([[landmarks[idx].x, landmarks[idx].y] for idx in [33, 133]], axis=0)
    face_center = (left_c + right_c) / 2
    landmarks_centered = (landmarks_arr - face_center).flatten()
    landmarks_tensor = torch.from_numpy(landmarks_centered).float().unsqueeze(0).to(device)
    
    # Prepare patches (using left eye for inference)
    left_eye_img, _ = preprocessor.normalize_eye(frame, landmarks, 'left')
    patches = preprocessor.extract_patches(left_eye_img, patch_size=16)
    patches_tensor = torch.from_numpy(patches).float().unsqueeze(0).to(device) / 255.0
    
    # Inference
    with torch.no_grad():
        p_logits, y_logits, _ = model(patches_tensor, landmarks_tensor)
        idx = torch.arange(90).float().to(device)
        pitch = (torch.sum(torch.softmax(p_logits, dim=1) * idx, dim=1) * 2 - 90).item()
        yaw = (torch.sum(torch.softmax(y_logits, dim=1) * idx, dim=1) * 2 - 90).item()
    
    # Draw
    res_frame = draw_gaze(frame.copy(), pitch, yaw, landmarks)
    res_img = cv2.cvtColor(res_frame, cv2.COLOR_BGR2RGB)
    
    return Image.fromarray(res_img), f"Pitch: {pitch:.2f}°, Yaw: {yaw:.2f}°"

# 3. Gradio Interface
description = """
### LIPE V2: Ultra-Lightweight Gaze Estimation
LIPE (Landmark-guided Image Patch Embedder) is a high-efficiency gaze estimation framework optimized for commodity CPUs.
- **Model Size:** 0.61M Parameters
- **Complexity:** 21.44 MFLOPs
- **Accuracy:** 4.73° (MPII), 7.33° (Gaze360 Zero-shot)
- **Speed:** 30 FPS on Single-threaded Mobile CPU
"""

iface = gr.Interface(
    fn=predict,
    inputs=gr.Image(type="pil"),
    outputs=[gr.Image(label="Gaze Visualization"), gr.Textbox(label="Result")],
    title="LIPE V2 Live Demo",
    description=description,
    examples=[["data/verification/sample_face.jpg"]] if os.path.exists("data/verification/sample_face.jpg") else None
)

if __name__ == "__main__":
    iface.launch()