Update app.py

app.py

@@ -1,51 +1,80 @@
-import gradio as gr
+import os
+import json
+import io
+
+import cv2
 import numpy as np
 import mediapipe as mp
-import cv2
+import tensorflow as tf
 from tensorflow import keras
-import json
-import os
 
-# === LOAD MODEL & LABELS ===
-MODEL_PATH = "models/sign_model_lstm_v1.keras"
-LABELS_PATH = "models/label_names.json"
+import gradio as gr
+import matplotlib.pyplot as plt
+from PIL import Image
 
-print(f"TensorFlow version: {keras.__version__}")
-print("Cargando modelo desde:", MODEL_PATH)
+print("TensorFlow version:", tf.__version__)
+
+# ==== RUTAS DEL MODELO ====
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+MODELS_DIR = os.path.join(BASE_DIR, "models")
 
+MODEL_PATH = os.path.join(MODELS_DIR, "sign_model_lstm_v1.keras")
+LABELS_PATH = os.path.join(MODELS_DIR, "label_names.json")
+
+print("Cargando modelo desde:", MODEL_PATH)
 model = keras.models.load_model(MODEL_PATH)
 
 with open(LABELS_PATH, "r") as f:
     label_names = json.load(f)
 
-NUM_POINTS = (33 + 21 + 21) * 3  # pose + left + right = (33 + 21 + 21) landmarks, xyz values
+num_classes = len(label_names)
+MAX_FRAMES = 20  # mismo valor que usaste al entrenar
 
+# ==== MEDIAPIPE ====
 mp_holistic = mp.solutions.holistic
+mp_drawing = mp.solutions.drawing_utils
+mp_styles = mp.solutions.drawing_styles
+
 
+# ---- 1. EXTRAER LANDMARKS COMO VECTOR (IGUAL QUE EN EL ENTRENAMIENTO) ----
 def extract_landmarks_from_results(results):
-    def get_xyz(landmarks, n):
+    """
+    Convierte los resultados de MediaPipe Holistic en un vector 1D (225,)
+    con pose (33), mano izq (21) y mano der (21).
+    Cada punto = (x, y, z) => 75 * 3 = 225 features.
+    """
+    def get_xyz(landmarks, n_points):
         if landmarks is None:
-            data = [[0,0,0]] * n
+            data = [[0.0, 0.0, 0.0]] * n_points
         else:
             data = [[lm.x, lm.y, lm.z] for lm in landmarks]
-            if len(data) < n:
-                data += [[0,0,0]] * (n - len(data))
-            data = data[:n]
+            if len(data) < n_points:
+                data += [[0.0, 0.0, 0.0]] * (n_points - len(data))
+            data = data[:n_points]
         return data
 
     pose = get_xyz(results.pose_landmarks.landmark if results.pose_landmarks else None, 33)
-    left = get_xyz(results.left_hand_landmarks.landmark if results.left_hand_landmarks else None, 21)
-    right = get_xyz(results.right_hand_landmarks.landmark if results.right_hand_landmarks else None, 21)
+    left_hand = get_xyz(results.left_hand_landmarks.landmark if results.left_hand_landmarks else None, 21)
+    right_hand = get_xyz(results.right_hand_landmarks.landmark if results.right_hand_landmarks else None, 21)
 
-    all_points = pose + left + right
-    return np.array(all_points, dtype=np.float32).flatten()
+    all_points = pose + left_hand + right_hand
+    return np.array(all_points, dtype=np.float32).flatten()  # (225,)
 
-def predict_video_lstm(video_path):
-    print("Procesando video:", video_path)
 
+# ---- 2. PROCESAR VIDEO -> SECUENCIA + FRAME CON LANDMARKS ----
+def video_to_sequence_and_landmarks_frame(video_path, max_frames=MAX_FRAMES):
+    """
+    Procesa un video:
+      - Devuelve la secuencia (1, max_frames, 225) para el LSTM
+      - Devuelve una imagen (PIL.Image) con los landmarks dibujados
+        en el primer frame donde se detecte algo.
+    """
     cap = cv2.VideoCapture(video_path)
-    frames_landmarks = []
-    
+
+    frames_feats = []
+    frame_for_vis = None
+    results_for_vis = None
+
     with mp_holistic.Holistic(
         static_image_mode=False,
         model_complexity=1,
@@ -60,54 +89,130 @@ def predict_video_lstm(video_path):
             if not ret:
                 break
 
-            image_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
-            results = holistic.process(image_rgb)
+            frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            results = holistic.process(frame_rgb)
 
             vec = extract_landmarks_from_results(results)
-            frames_landmarks.append(vec)
+            frames_feats.append(vec)
+
+            # Guardamos el primer frame donde se detecta algo para visualizar
+            if frame_for_vis is None and (
+                results.pose_landmarks or
+                results.left_hand_landmarks or
+                results.right_hand_landmarks
+            ):
+                frame_for_vis = frame.copy()
+                results_for_vis = results
+
+            if len(frames_feats) >= max_frames:
+                break
 
     cap.release()
 
-    if len(frames_landmarks) == 0:
-        return "No se encontraron landmarks", {}
+    if len(frames_feats) == 0:
+        raise ValueError("No se pudieron leer frames válidos del video.")
 
-    # Pad / slice to fixed length (20 frames)
-    MAX_FRAMES = 20
-    if len(frames_landmarks) < MAX_FRAMES:
-        pad = [np.zeros(NUM_POINTS)] * (MAX_FRAMES - len(frames_landmarks))
-        frames_landmarks = frames_landmarks + pad
+    seq = np.array(frames_feats, dtype=np.float32)
+
+    # Padding / recorte
+    if seq.shape[0] < max_frames:
+        pad_len = max_frames - seq.shape[0]
+        pad = np.zeros((pad_len, seq.shape[1]), dtype=np.float32)
+        seq = np.concatenate([seq, pad], axis=0)
+    else:
+        seq = seq[:max_frames, :]
+
+    seq = seq.reshape(1, max_frames, seq.shape[1])  # (1, T, 225)
+
+    # --- crear imagen con landmarks ---
+    landmarks_image = None
+    if frame_for_vis is not None and results_for_vis is not None:
+        annotated = frame_for_vis.copy()
+
+        mp_drawing.draw_landmarks(
+            annotated,
+            results_for_vis.pose_landmarks,
+            mp_holistic.POSE_CONNECTIONS,
+            landmark_drawing_spec=mp_styles.get_default_pose_landmarks_style()
+        )
+        mp_drawing.draw_landmarks(
+            annotated,
+            results_for_vis.left_hand_landmarks,
+            mp_holistic.HAND_CONNECTIONS,
+            landmark_drawing_spec=mp_styles.get_default_hand_landmarks_style()
+        )
+        mp_drawing.draw_landmarks(
+            annotated,
+            results_for_vis.right_hand_landmarks,
+            mp_holistic.HAND_CONNECTIONS,
+            landmark_drawing_spec=mp_styles.get_default_hand_landmarks_style()
+        )
+
+        annotated_rgb = cv2.cvtColor(annotated, cv2.COLOR_BGR2RGB)
+        landmarks_image = Image.fromarray(annotated_rgb)
+
+    return seq, landmarks_image
+
+
+# ---- 3. PREDICCIÓN + GRÁFICO ----
+def predict_from_video(video):
+    """
+    Función que usa Gradio:
+      - recibe ruta del video (upload o webcam)
+      - devuelve: texto con predicción, plot de barras, imagen con landmarks
+    """
+    if video is None:
+        return "Sube un video o grábalo primero.", None, None
+
+    # Gradio pasa un dict con la ruta en 'video'
+    if isinstance(video, dict) and "video" in video:
+        video_path = video["video"]
     else:
-        frames_landmarks = frames_landmarks[:MAX_FRAMES]
+        video_path = video
 
-    X = np.array(frames_landmarks, dtype=np.float32).reshape(1, MAX_FRAMES, NUM_POINTS)
+    seq, landmarks_image = video_to_sequence_and_landmarks_frame(video_path, MAX_FRAMES)
 
-    probs = model.predict(X, verbose=0)[0]
+    probs = model.predict(seq, verbose=0)[0]  # (num_classes,)
     idx = int(np.argmax(probs))
-    
-    prediction = label_names[idx]
-    confidences = {label_names[i]: float(probs[i]) for i in range(len(probs))}
-
-    return f"Predicción: {prediction}", confidences
-
-# === GRADIO UI ===
+    label = label_names[idx]
+    conf = float(probs[idx])
+
+    # Texto de salida
+    text_pred = f"Predicción: {label} (confianza {conf:.2f})"
+
+    # Gráfico de barras con las probabilidades
+    fig, ax = plt.subplots(figsize=(6, 3))
+    ax.bar(range(len(label_names)), probs)
+    ax.set_xticks(range(len(label_names)))
+    ax.set_xticklabels(label_names, rotation=45, ha="right")
+    ax.set_ylim(0, 1)
+    ax.set_ylabel("Confianza")
+    ax.set_title("Confianza por clase")
+    plt.tight_layout()
+
+    return text_pred, fig, landmarks_image
+
+
+# ---- 4. INTERFAZ GRADIO ----
+title = "LSP-EnSeñas - Demo LSTM"
+description = (
+    "Traductor de señas basado en LSTM + MediaPipe Holistic. "
+    "Sube un video corto o grábalo en vivo haciendo una seña. "
+    "El modelo procesará el movimiento (cuerpo + manos) y mostrará la predicción. "
+    "También verás un frame con los puntos (landmarks) detectados por MediaPipe."
+)
 
 demo = gr.Interface(
-    fn=predict_video_lstm,
-    inputs=gr.Video(
-        sources=["upload", "webcam"],
-        label="Sube un video o graba tu seña",
-        format="mp4"
-    ),
+    fn=predict_from_video,
+    inputs=gr.Video(label="Sube un video o grábalo desde la cámara"),
     outputs=[
-        gr.Textbox(label="Resultado"),
-        gr.Label(label="Confianza por clase")
+        gr.Textbox(label="Predicción del modelo"),
+        gr.Plot(label="Confianza por clase"),
+        gr.Image(type="pil", label="Landmarks detectados (ejemplo de frame)")
     ],
-    title="LSP-EnSeñas - Demo LSTM",
-    description=(
-        "Traductor de señas basado en LSTM + MediaPipe Holistic.\n"
-        "Sube un video corto o grábalo en vivo haciendo una seña.\n"
-        "El modelo procesará el movimiento (cuerpo + manos) y mostrará la predicción."
-    )
+    title=title,
+    description=description,
 )
 
-demo.launch()
+if __name__ == "__main__":
+    demo.launch()