Update app.py

app.py

@@ -1,87 +1,101 @@
 import os
 import json
-import cv2
 import numpy as np
+import cv2
+import gradio as gr
 import mediapipe as mp
 import tensorflow as tf
 from tensorflow import keras
 
-import gradio as gr
-import matplotlib.pyplot as plt
+# ---------------------------------------------------------
+# CONFIG
+# ---------------------------------------------------------
+MODELS_DIR = "models"
+MAX_FRAMES = 20
+N_FEATURES = 225  # 75 landmarks * (x,y,z)
 
-print("TensorFlow version:", tf.__version__)
+mp_holistic = mp.solutions.holistic
 
-# ==== RUTAS DEL MODELO ====
-BASE_DIR = os.path.dirname(os.path.abspath(__file__))
-MODELS_DIR = os.path.join(BASE_DIR, "models")
+# ---------------------------------------------------------
+# LANDMARK EXTRACTION (MISMO QUE EN TRAIN)
+# ---------------------------------------------------------
+def extract_landmarks_from_results(results):
+    """
+    Devuelve un vector plano de 225 floats (75 puntos * 3 coords).
+    """
+    # Order: left hand (21), right hand (21), pose (33)
+    final = []
 
-MODEL_PATH = os.path.join(MODELS_DIR, "sign_model_lstm_v1.keras")
-LABELS_PATH = os.path.join(MODELS_DIR, "label_names.json")
+    # LEFT HAND
+    if results.left_hand_landmarks:
+        for lm in results.left_hand_landmarks.landmark:
+            final.extend([lm.x, lm.y, lm.z])
+    else:
+        final.extend([0.0] * 63)
 
-print("Cargando modelo desde:", MODEL_PATH)
-model = keras.models.load_model(MODEL_PATH)
+    # RIGHT HAND
+    if results.right_hand_landmarks:
+        for lm in results.right_hand_landmarks.landmark:
+            final.extend([lm.x, lm.y, lm.z])
+    else:
+        final.extend([0.0] * 63)
 
-with open(LABELS_PATH, "r") as f:
-    label_names = json.load(f)
+    # POSE (33 puntos)
+    if results.pose_landmarks:
+        for lm in results.pose_landmarks.landmark:
+            final.extend([lm.x, lm.y, lm.z])
+    else:
+        final.extend([0.0] * 99)
 
-num_classes = len(label_names)
-MAX_FRAMES = 20  # mismo valor que usaste al entrenar
+    return np.array(final, dtype=np.float32)
 
 
-# ==== MEDIAPIPE ====
-mp_holistic = mp.solutions.holistic
-mp_drawing = mp.solutions.drawing_utils
-mp_styles = mp.solutions.drawing_styles
+# ---------------------------------------------------------
+# PAD/TRUNCATE EXACTO AL DEL NOTEBOOK
+# ---------------------------------------------------------
+def pad_or_truncate(seq, max_frames=MAX_FRAMES):
+    T = seq.shape[0]
+    if T == max_frames:
+        return seq
+    elif T > max_frames:
+        start = (T - max_frames) // 2
+        return seq[start:start + max_frames]
+    else:
+        pad_len = max_frames - T
+        pad = np.zeros((pad_len, seq.shape[1]), dtype=np.float32)
+        return np.concatenate([seq, pad], axis=0)
 
 
-def extract_landmarks_from_results(results):
-    """
-    Convierte los resultados de MediaPipe Holistic en un vector 1D (225,)
-    con pose (33), mano izq (21) y mano der (21).
-    Cada punto = (x, y, z) => 75 * 3 = 225 features.
-    """
-    def get_xyz(landmarks, n_points):
-        if landmarks is None:
-            data = [[0.0, 0.0, 0.0]] * n_points
-        else:
-            data = [[lm.x, lm.y, lm.z] for lm in landmarks]
-            if len(data) < n_points:
-                data += [[0.0, 0.0, 0.0]] * (n_points - len(data))
-            data = data[:n_points]
-        return data
+# ---------------------------------------------------------
+# CARGAR MODELO + LABELS + NORMALIZACIÓN
+# ---------------------------------------------------------
+def load_model():
+    model_path = os.path.join(MODELS_DIR, "sign_model_lstm_v1.keras")
+    labels_path = os.path.join(MODELS_DIR, "label_names.json")
+    mean_path = os.path.join(MODELS_DIR, "feature_mean.npy")
+    std_path = os.path.join(MODELS_DIR, "feature_std.npy")
 
-    pose = get_xyz(results.pose_landmarks.landmark if results.pose_landmarks else None, 33)
-    left_hand = get_xyz(results.left_hand_landmarks.landmark if results.left_hand_landmarks else None, 21)
-    right_hand = get_xyz(results.right_hand_landmarks.landmark if results.right_hand_landmarks else None, 21)
+    model = keras.models.load_model(model_path)
 
-    all_points = pose + left_hand + right_hand
-    return np.array(all_points, dtype=np.float32).flatten()  # (225,)
+    with open(labels_path, "r") as f:
+        label_names = json.load(f)
 
+    feature_mean = np.load(mean_path)
+    feature_std = np.load(std_path)
+
+    return model, label_names, feature_mean, feature_std
 
-def video_to_sequence_and_annotated(video_path, max_frames=MAX_FRAMES):
-    """
-    Procesa un video:
-      - Devuelve la secuencia (1, max_frames, 225) para el LSTM
-      - Devuelve la ruta de un nuevo video con los landmarks dibujados.
-    """
-    cap = cv2.VideoCapture(video_path)
 
-    if not cap.isOpened():
-        raise ValueError(f"No se pudo abrir el video: {video_path}")
+model, label_names, feature_mean, feature_std = load_model()
 
-    frames_feats = []
 
-    # Info del video de entrada
-    width  = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
-    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
-    fps    = cap.get(cv2.CAP_PROP_FPS)
-    if fps is None or fps <= 0:
-        fps = 25.0
+# ---------------------------------------------------------
+# PROCESAR VIDEO (MISMO QUE EN TRAIN)
+# ---------------------------------------------------------
+def process_video(video_file):
 
-    # Ruta temporal para el video anotado
-    out_path = os.path.join("/tmp", "annotated_output.mp4")
-    fourcc = cv2.VideoWriter_fourcc(*"mp4v")
-    writer = cv2.VideoWriter(out_path, fourcc, fps, (width, height))
+    cap = cv2.VideoCapture(video_file)
+    frames = []
 
     with mp_holistic.Holistic(
         static_image_mode=False,
@@ -90,128 +104,61 @@ def video_to_sequence_and_annotated(video_path, max_frames=MAX_FRAMES):
         refine_face_landmarks=False,
         min_detection_confidence=0.5,
         min_tracking_confidence=0.5
-    ) as holistic:
+    ) as holis:
 
-        frame_idx = 0
         while True:
             ret, frame = cap.read()
             if not ret:
                 break
 
-            frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
-            results = holistic.process(frame_rgb)
-
-            # Features para el modelo (solo guardamos hasta max_frames)
-            if len(frames_feats) < max_frames:
-                vec = extract_landmarks_from_results(results)
-                frames_feats.append(vec)
-
-            # DIBUJAR LANDMARKS SOBRE EL FRAME
-            annotated = frame.copy()
-
-            if results.pose_landmarks:
-                mp_drawing.draw_landmarks(
-                    annotated,
-                    results.pose_landmarks,
-                    mp_holistic.POSE_CONNECTIONS,
-                    landmark_drawing_spec=mp_styles.get_default_pose_landmarks_style()
-                )
-            if results.left_hand_landmarks:
-                mp_drawing.draw_landmarks(
-                    annotated,
-                    results.left_hand_landmarks,
-                    mp_holistic.HAND_CONNECTIONS,
-                    landmark_drawing_spec=mp_styles.get_default_hand_landmarks_style()
-                )
-            if results.right_hand_landmarks:
-                mp_drawing.draw_landmarks(
-                    annotated,
-                    results.right_hand_landmarks,
-                    mp_holistic.HAND_CONNECTIONS,
-                    landmark_drawing_spec=mp_styles.get_default_hand_landmarks_style()
-                )
-
-            writer.write(annotated)
-            frame_idx += 1
+            rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            results = holis.process(rgb)
+            feats = extract_landmarks_from_results(results)
+            frames.append(feats)
 
     cap.release()
-    writer.release()
 
-    if len(frames_feats) == 0:
-        raise ValueError("No se pudieron leer frames válidos del video.")
-
-    seq = np.array(frames_feats, dtype=np.float32)
-
-    # Padding / recorte para el LSTM
-    if seq.shape[0] < max_frames:
-        pad_len = max_frames - seq.shape[0]
-        pad = np.zeros((pad_len, seq.shape[1]), dtype=np.float32)
-        seq = np.concatenate([seq, pad], axis=0)
+    if len(frames) == 0:
+        seq = np.zeros((MAX_FRAMES, N_FEATURES), dtype=np.float32)
     else:
-        seq = seq[:max_frames, :]
+        seq_full = np.stack(frames, axis=0)
+        seq = pad_or_truncate(seq_full)
 
-    seq = seq.reshape(1, max_frames, seq.shape[1])  # (1, T, 225)
+    seq = seq[np.newaxis, ...]  # (1, T, 225)
 
-    return seq, out_path
+    # NORMALIZACIÓN IGUAL
+    seq_norm = (seq - feature_mean) / feature_std
 
+    return seq_norm
 
-def predict_from_video(video):
-    """
-    Función llamada por Gradio.
-      - Recibe ruta del video (upload o webcam).
-      - Devuelve: texto, gráfico de barras, video con landmarks.
-    """
-    if video is None:
-        return "Sube un video o grábalo primero.", None, None
-
-    # Gradio a veces pasa dict con 'video'
-    if isinstance(video, dict) and "video" in video:
-        video_path = video["video"]
-    else:
-        video_path = video
-
-    seq, annotated_path = video_to_sequence_and_annotated(video_path, MAX_FRAMES)
 
-    probs = model.predict(seq, verbose=0)[0]  # (num_classes,)
+# ---------------------------------------------------------
+# PREDICCIÓN FINAL
+# ---------------------------------------------------------
+def predict(video):
+    seq = process_video(video)
+    probs = model.predict(seq, verbose=0)[0]
     idx = int(np.argmax(probs))
     label = label_names[idx]
     conf = float(probs[idx])
 
-    # Texto
-    text_pred = f"Predicción: {label} (confianza {conf:.2f})"
-
-    # Gráfico de barras
-    fig, ax = plt.subplots(figsize=(6, 3))
-    ax.bar(range(len(label_names)), probs)
-    ax.set_xticks(range(len(label_names)))
-    ax.set_xticklabels(label_names, rotation=45, ha="right")
-    ax.set_ylim(0, 1)
-    ax.set_ylabel("Confianza")
-    ax.set_title("Confianza por clase")
-    plt.tight_layout()
-
-    # annotated_path es la ruta del video con landmarks
-    return text_pred, fig, annotated_path
-
-
-title = "LSP-EnSeñas - Demo LSTM"
-description = (
-    "Traductor de señas basado en LSTM + MediaPipe Holistic. "
-    "Sube un video corto o grábalo en vivo haciendo una seña. "
-    "El modelo procesará el movimiento (cuerpo + manos), mostrará la predicción "
-    "y devolverá tu video con los puntos (landmarks) dibujados."
-)
+    # Formato bonito para Gradio
+    probs_dict = {label_names[i]: float(probs[i]) for i in range(len(label_names))}
+
+    return f"Predicción: {label} (confianza {conf:.2f})", probs_dict
+
 
+# ---------------------------------------------------------
+# UI GRADIO
+# ---------------------------------------------------------
 demo = gr.Interface(
-    fn=predict_from_video,
-    inputs=gr.Video(label="Sube un video o grábalo desde la cámara"),
+    fn=predict,
+    inputs=gr.Video(label="Sube un video haciendo la seña"),
     outputs=[
-        gr.Textbox(label="Predicción del modelo"),
-        gr.Plot(label="Confianza por clase"),
-        gr.Video(label="Video con landmarks detectados"),
+        gr.Textbox(label="Resultado"),
+        gr.Label(label="Probabilidades por clase")
     ],
-    title=title,
-    description=description,
+    title="Sign Language Translator - LSTM"
 )
 
 if __name__ == "__main__":