Update app.py

app.py

@@ -4,6 +4,8 @@ import numpy as np
 import torch
 import torch.nn as nn
 import gradio as gr
+import tempfile
+import os
 
 # ----------------------------
 # Labels
@@ -45,18 +47,13 @@ model.eval()
 # MediaPipe
 # ----------------------------
 mp_hands = mp.solutions.hands
+mp_drawing = mp.solutions.drawing_utils
+mp_drawing_styles = mp.solutions.drawing_styles
 
 # ----------------------------
-# Predict
+# Core landmark extraction + prediction
 # ----------------------------
-def predict(image):
-    # Gradio sends RGB, convert to BGR for OpenCV then back to RGB for MediaPipe
-    image_bgr = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
-    image_rgb = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)
-
-    with mp_hands.Hands(static_image_mode=True, max_num_hands=2) as hands:
-        results = hands.process(image_rgb)
-
+def extract_coords(results):
     coords = []
     if results.multi_hand_landmarks:
         for hand_landmarks in results.multi_hand_landmarks:
@@ -66,32 +63,144 @@ def predict(image):
             if max_val > 0:
                 hand_coords /= max_val
             coords.extend(hand_coords.flatten())
+    return coords
+
 
+def run_model(coords):
+    """Return (label, confidence) or ('Unknown', 0.0)."""
     if len(coords) < 126:
-        coords.extend([0.0] * (126 - len(coords)))
+        coords = coords + [0.0] * (126 - len(coords))
     elif len(coords) > 126:
         coords = coords[:126]
 
-    if len(coords) == 126:
-        input_tensor = torch.tensor(coords, dtype=torch.float32).unsqueeze(0)
-        with torch.no_grad():
-            outputs = model(input_tensor)
-            probs = torch.softmax(outputs, dim=1)
-            pred_class = torch.argmax(probs, dim=1).item()
-            confidence = probs[0][pred_class].item()
-        if confidence >= CONF_THRESHOLD:
-            return f"{GESTURE_LABELS[pred_class]} ({confidence*100:.2f}%)"
+    input_tensor = torch.tensor(coords, dtype=torch.float32).unsqueeze(0)
+    with torch.no_grad():
+        outputs = model(input_tensor)
+        probs = torch.softmax(outputs, dim=1)
+        pred_class = torch.argmax(probs, dim=1).item()
+        confidence = probs[0][pred_class].item()
+
+    if confidence >= CONF_THRESHOLD:
+        return GESTURE_LABELS[pred_class], confidence
+    return "Unknown", confidence
 
-    return "Unknown"
 
 # ----------------------------
-# Gradio UI
+# Image prediction
 # ----------------------------
-app = gr.Interface(
-    fn=predict,
-    inputs=gr.Image(type="numpy"),
-    outputs="text",
-    title="Hand Gesture Recognition",
-    description="Upload an image of a hand gesture to recognize it."
-)
+def predict_image(image):
+    image_rgb = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+    image_rgb = cv2.cvtColor(image_rgb, cv2.COLOR_BGR2RGB)
+
+    with mp_hands.Hands(static_image_mode=True, max_num_hands=2) as hands:
+        results = hands.process(image_rgb)
+
+    coords = extract_coords(results)
+    if not coords:
+        return "No hand detected"
+
+    label, conf = run_model(coords)
+    return f"{label} ({conf*100:.2f}%)"
+
+
+# ----------------------------
+# Video prediction
+# ----------------------------
+def predict_video(video_path):
+    if video_path is None:
+        return None
+
+    cap = cv2.VideoCapture(video_path)
+    if not cap.isOpened():
+        return None
+
+    fps = cap.get(cv2.CAP_PROP_FPS) or 25
+    width  = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+
+    # Write to a temp file that Gradio can serve
+    tmp = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False)
+    out_path = tmp.name
+    tmp.close()
+
+    fourcc = cv2.VideoWriter_fourcc(*"mp4v")
+    writer = cv2.VideoWriter(out_path, fourcc, fps, (width, height))
+
+    with mp_hands.Hands(
+        static_image_mode=False,
+        max_num_hands=2,
+        min_detection_confidence=0.5,
+        min_tracking_confidence=0.5,
+    ) as hands:
+        while True:
+            ret, frame = cap.read()
+            if not ret:
+                break
+
+            frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            results = hands.process(frame_rgb)
+
+            # Draw landmarks
+            if results.multi_hand_landmarks:
+                for hand_landmarks in results.multi_hand_landmarks:
+                    mp_drawing.draw_landmarks(
+                        frame,
+                        hand_landmarks,
+                        mp_hands.HAND_CONNECTIONS,
+                        mp_drawing_styles.get_default_hand_landmarks_style(),
+                        mp_drawing_styles.get_default_hand_connections_style(),
+                    )
+
+            # Predict and overlay label
+            coords = extract_coords(results)
+            if coords:
+                label, conf = run_model(coords)
+                text = f"{label} ({conf*100:.1f}%)"
+                color = (0, 220, 0) if label != "Unknown" else (0, 0, 220)
+            else:
+                text = "No hand detected"
+                color = (180, 180, 180)
+
+            cv2.putText(
+                frame, text,
+                (20, 50),
+                cv2.FONT_HERSHEY_SIMPLEX,
+                1.4, color, 3, cv2.LINE_AA,
+            )
+
+            writer.write(frame)
+
+    cap.release()
+    writer.release()
+    return out_path
+
+
+# ----------------------------
+# Gradio UI  –  two tabs
+# ----------------------------
+with gr.Blocks(title="Hand Gesture Recognition") as app:
+    gr.Markdown("# ✋ Hand Gesture Recognition")
+    gr.Markdown(
+        "Recognises **25 gestures** (A B E F L U V W Z i k m r s "
+        "zero one two three four five six seven eight nine ten)."
+    )
+
+    with gr.Tab("📷 Image"):
+        with gr.Row():
+            img_input  = gr.Image(type="numpy", label="Upload image")
+            img_output = gr.Textbox(label="Prediction")
+        img_btn = gr.Button("Predict", variant="primary")
+        img_btn.click(predict_image, inputs=img_input, outputs=img_output)
+
+    with gr.Tab("🎬 Video"):
+        gr.Markdown(
+            "Upload a short video clip. Each frame is processed and the "
+            "predicted gesture is overlaid. Hand landmarks are drawn in real time."
+        )
+        with gr.Row():
+            vid_input  = gr.Video(label="Upload video")
+            vid_output = gr.Video(label="Annotated output")
+        vid_btn = gr.Button("Predict", variant="primary")
+        vid_btn.click(predict_video, inputs=vid_input, outputs=vid_output)
+
 app.launch()