app.py

import gradio as gr
import cv2
import numpy as np
import tempfile
import json
from modules import mediapipe_generator
from modules.mediapipe_generator import extract_to_array
from modules.sentence_guesser import SentenceGuesser

guesser = SentenceGuesser(model="fold1")

EXTRACTION_PATH = "metadata/extraction_order.json"
SIGN_DICT_PATH = "metadata/sign_dict.json"

with open(EXTRACTION_PATH, 'r') as f:
    extraction_order = json.load(f)
with open(SIGN_DICT_PATH, 'r') as f:
    sign_dict = {int(k): v for k, v in json.load(f).items()}

def retry_tts(cached_words, cached_emotions):
    if not cached_words or not cached_emotions:
        gr.Warning("No translation available to synthesize yet. Please analyze a sequence first.")
        return None
    
    print("🔄 Retrying TTS Generation...")
    #return generate_speech(cached_words, cached_emotions)
    return None

def get_sign_label(idx):
    if idx == 80:
        return ""
    else:
        return sign_dict.get(idx, f"ID_{idx}")

def render_normalized_component(landmarks, canvas_w, canvas_h, padding=0.1, color=(0, 255, 0)):
    canvas = np.zeros((canvas_h, canvas_w, 3), dtype=np.uint8)
    
    if landmarks.shape[0] == 0 or np.isnan(landmarks).any():
        return canvas
        
    min_x, max_x = np.min(landmarks[:, 0]), np.max(landmarks[:, 0])
    min_y, max_y = np.min(landmarks[:, 1]), np.max(landmarks[:, 1])
    
    data_w = max_x - min_x
    data_h = max_y - min_y
    
    if data_w == 0 or data_h == 0:
        return canvas
        
    usable_w = canvas_w * (1 - padding * 2)
    usable_h = canvas_h * (1 - padding * 2)
    scale = min(usable_w / data_w, usable_h / data_h)
    
    cx, cy = (min_x + max_x) / 2, (min_y + max_y) / 2
    canvas_cx, canvas_cy = canvas_w / 2, canvas_h / 2
    
    for x, y in landmarks:
        pix_x = int((x - cx) * scale + canvas_cx)
        pix_y = int((y - cy) * scale + canvas_cy)
        cv2.circle(canvas, (pix_x, pix_y), 2, color, -1)
        
    return canvas

def render_diagnostic_matrix(orig_frame, raw_landmarks, norm_pose, norm_lhand, norm_rhand, norm_face, current_label):
    cell_w, cell_h = 320, 240
    
    raw_rgb = cv2.resize(orig_frame, (cell_w, cell_h))
    raw_overlaid = raw_rgb.copy()
    
    for lm in raw_landmarks:
        x_pix, y_pix = int(lm[0] * cell_w), int(lm[1] * cell_h)
        cv2.circle(raw_overlaid, (x_pix, y_pix), 2, (0, 0, 255), -1)
        
    canvas_pose = render_normalized_component(norm_pose, cell_w, cell_h, color=(255, 255, 0))
    canvas_face = render_normalized_component(norm_face, cell_w, cell_h, color=(0, 255, 255))
    canvas_lhand = render_normalized_component(norm_lhand, cell_w, cell_h, color=(255, 0, 255))
    canvas_rhand = render_normalized_component(norm_rhand, cell_w, cell_h, color=(0, 255, 0))

    def add_title(img, text):
        cv2.putText(img, text, (10, 25), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 1)
        return img

    add_title(raw_rgb, "Raw RGB")
    cv2.putText(raw_rgb, f"PRED: {current_label}", (10, cell_h - 15), cv2.FONT_HERSHEY_DUPLEX, 0.7, (0, 255, 0), 2)
    
    add_title(raw_overlaid, "Raw + Landmarks")
    add_title(canvas_pose, "Norm: Pose")
    add_title(canvas_lhand, "Norm: L-Hand")
    add_title(canvas_rhand, "Norm: R-Hand")
    add_title(canvas_face, "Norm: Face")

    row1 = np.hstack((raw_rgb, raw_overlaid))
    row2 = np.hstack((canvas_pose, canvas_face))
    row3 = np.hstack((canvas_lhand, canvas_rhand))
    
    return np.vstack((row1, row2, row3))

def render_diagnostic_video(original_path, raw_data, norm_data, frame_indices, fps=30):
    temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.mp4')
    output_path = temp_file.name
    
    w, h = 640, 720 
    fourcc = cv2.VideoWriter_fourcc(*'mp4v')
    out = cv2.VideoWriter(output_path, fourcc, fps, (w, h))
    cap = cv2.VideoCapture(original_path)

    for t in range(raw_data.shape[0]):
        ret, orig_frame = cap.read()
        if not ret:
            orig_frame = np.zeros((480, 640, 3), dtype=np.uint8)

        norm_pose = norm_data[t, :7, :]
        norm_lhand = norm_data[t, 7:28, :]
        norm_rhand = norm_data[t, 28:49, :]
        norm_face = norm_data[t, 49:, :]
        
        current_label = get_sign_label(int(frame_indices[t]))

        matrix_frame = render_diagnostic_matrix(
            orig_frame, 
            raw_data[t], 
            norm_pose, 
            norm_lhand, 
            norm_rhand, 
            norm_face,
            current_label
        )
        out.write(matrix_frame)

    cap.release()
    out.release()
    return output_path

def predict_sign(video_path):    
    mediapipe_results = mediapipe_generator.generate_mediapipe_gradio(video_path)
    pose_seq = np.array([extract_to_array(r.pose_landmarks, 33, 4) for r in mediapipe_results])
    face_seq = np.array([extract_to_array(r.face_landmarks, 468, 3) for r in mediapipe_results])
    lh_seq = np.array([extract_to_array(r.left_hand_landmarks, 21, 3) for r in mediapipe_results])
    rh_seq = np.array([extract_to_array(r.right_hand_landmarks, 21, 3) for r in mediapipe_results])

    result = guesser.predict(pose_seq, face_seq, lh_seq, rh_seq)

    words = result['prediction_string'].split()
    emotions = result['prediction_emotions']
    
    mapped_output = "\n".join([f"{w} -> [{e}]" for w, e in zip(words, emotions)])

    display_text = (
        f"Raw Window IDs: {result['raw_ids']}\n\n"
        f"Translation: {result['prediction_string']}\n\n"
        f"Emotion Mapping:\n{mapped_output}"
    )
    viz_path = render_diagnostic_video(
        video_path, 
        result['raw_data'], 
        result['norm_data'], 
        result['frame_indices']
    )

    # audio_path = generate_speech(words, emotions)
    audio_path = None
    
    return display_text, audio_path, viz_path, words, emotions

with gr.Blocks() as demo:
    current_words = gr.State([])
    current_emotions = gr.State([])

    gr.Markdown("# Filipino Sign Language Recognition")

    with gr.Row():
        with gr.Column():
            video_input = gr.Video(label="Input: Upload or Record")
            submit_btn = gr.Button("Analyze Sequence", variant="primary")
        
        with gr.Column():
            output_text = gr.Textbox(label="Model Predictions")

            viz_output = gr.Video(label="Detected and Normalized Landmarks")

            with gr.Row():
                # audio_output = gr.Audio(label="Synthesized Speech", autoplay=True, scale=3)
                retry_audio_btn = gr.Button("🔄 Retry Audio", size="sm", scale=1)

    submit_btn.click(
        fn=predict_sign, 
        inputs=video_input, 
        outputs=[output_text, audio_output, viz_output, current_words, current_emotions] 
    )
    
    retry_audio_btn.click(
        fn=retry_tts,
        inputs=[current_words, current_emotions],
        outputs=[audio_output]
    )

demo.launch()