import gradio as gr import matplotlib matplotlib.use('Agg') import cv2 import os import tempfile import base64 import numpy as np import torch import mediapipe as mp from model import CustomLSTM, STGCNModel, CTRGCNModel, SkateFormerModel from utils import mediapipe_detection, draw_styled_landmarks, extract_keypoints, get_adjacency_matrix, reshape_for_stgcn # --- Setup --- device = torch.device("cuda" if torch.cuda.is_available() else "cpu") gloss = np.load('gloss.npy') # Load your preferred model (Defaulting to SkateFormer based on your inference.py) model = SkateFormerModel(num_classes=len(gloss)).to(device) model.load_state_dict(torch.load('best_model.pth', map_location=device)) model.eval() mp_holistic = mp.solutions.holistic def process_video(video_path): if video_path is None: return None, "⚠️ Please record a video for prediction." try: cap = cv2.VideoCapture(video_path) fps = cap.get(cv2.CAP_PROP_FPS) or 30 width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) frames_for_display = [] sequence = [] with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic: while cap.isOpened(): ret, frame = cap.read() if not ret: break # Extract keypoints using your existing logic image, results = mediapipe_detection(frame, holistic) draw_styled_landmarks(image, results) keypoints = extract_keypoints(results) sequence.append(keypoints) frames_for_display.append(image) cap.release() # Padding/Sampling logic to ensure exactly 30 frames if len(sequence) < 30: sequence.extend([np.zeros(258)] * (30 - len(sequence))) else: # Uniformly sample 30 frames idx = np.linspace(0, len(sequence) - 1, 30).astype(int) sequence = [sequence[i] for i in idx] # Model Inference input_data = np.expand_dims(sequence, axis=0) input_processed = reshape_for_stgcn(input_data) input_tensor = torch.tensor(input_processed, dtype=torch.float32).to(device) with torch.no_grad(): res = model(input_tensor) prob = torch.nn.functional.softmax(res, dim=1) confidence, max_idx = torch.max(prob, dim=1) label = gloss[max_idx.item()] result_text = f"{label} ({confidence.item() * 100:.2f}%)" # Create a temporary file to save the video fd, output_path = tempfile.mkstemp(suffix='.mp4') os.close(fd) # Use MP4V for mp4 files; ensures browser compatibility fourcc = cv2.VideoWriter_fourcc(*'mp4v') out = cv2.VideoWriter(output_path, fourcc, fps, (width, height)) for frame in frames_for_display: # Draw background rectangle for readability cv2.rectangle(frame, (0, 0), (width, 60), (245, 117, 16), -1) # Put prediction text on frame cv2.putText(frame, result_text, (10, 45), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA) out.write(frame) out.release() return output_path, "✅ Analysis Complete" except Exception as e: return None, f"❌ Error: {str(e)}" def get_img_html(img_path): """ Reads a local image file and converts it to a Base64 HTML string. This prevents "broken image" errors in Gradio. """ try: with open(img_path, "rb") as image_file: encoded_string = base64.b64encode(image_file.read()).decode() # Return the HTML img tag with the data embedded return f'' except Exception as e: print(f"Could not load icon: {e}") return "" # Return empty string if file missing def reset_state(): return None, None, "Ready" # --- Gradio UI --- with gr.Blocks(title="eBIM-Satu (Beta Version)") as demo: icon_html = get_img_html("favicon.png") gr.Markdown(f"# {icon_html} eBIM-Satu") gr.Markdown(f"### Malaysia Isolated Sign Language Recognition") gr.Markdown("Click 'Record' to start. The system will process the sign after 3 seconds.") with gr.Row(): with gr.Column(scale=1): system_status = gr.Textbox(label="System Status", value="Ready", lines=1) gr.Markdown( "### Instructions\n1. Open your camera.\n2. Click the record button in the video box.\n3. Perform the sign clearly.\n4. Click 'Start Prediction'.") with gr.Column(scale=2): input_video = gr.Video(sources=["webcam"], format="mp4", label="Sign Camera", webcam_options=gr.WebcamOptions(mirror=False)) with gr.Row(): predict_btn = gr.Button("Start Prediction / Analyze", variant="primary", scale=2) clear_btn = gr.Button("Clear / Reset", variant="secondary", scale=1) with gr.Column(scale=2): output_video = gr.Video(autoplay=True, show_label=False) predict_btn.click( fn=process_video, inputs=input_video, outputs=[output_video, system_status] ) clear_btn.click( fn=reset_state, inputs=None, outputs=[input_video, output_video, system_status] ) demo.launch(theme=gr.themes.Soft(), favicon_path="favicon.png")