Upload 5 files

Lipnet.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dc9c92165acdfa5fffde1e002235d611407a132967cb1865545541a6479db46
+size 98068312

app.py

@@ -0,0 +1,119 @@
+import gradio as gr
+import tempfile
+import os
+import cv2
+import numpy as np
+from inference3 import predict_from_video, LipReadingModel
+import logging
+
+# Configure Logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Load the model once
+def load_model():
+    logger.info("Loading Lip Reading Model...")
+    return LipReadingModel()
+
+model = load_model()
+
+# Prediction function with enhancements
+def run_prediction(video_path):
+    """
+    Takes a video file path, processes it, and returns the predicted text.
+    Includes error handling.
+    """
+    MAX_SIZE_MB = 1000  # Maximum allowed video size in megabytes
+
+    if not video_path:
+        return "❌ No video provided. Please upload or record a video."
+
+    # Check video size
+    try:
+        video_size_mb = os.path.getsize(video_path) / (1024 * 1024)
+        logger.info(f"Uploaded video size: {video_size_mb:.2f} MB")
+    except Exception as e:
+        logger.error(f"Error accessing video file: {e}")
+        return f"❌ Error accessing video file: {e}"
+
+    if video_size_mb > MAX_SIZE_MB:
+        return f"❌ Video size exceeds {MAX_SIZE_MB} MB limit. Please upload a smaller video."
+
+    try:
+        # Run prediction
+        logger.info("Running prediction...")
+        prediction = predict_from_video(video_path=video_path, model=model)
+        logger.info("Prediction completed.")
+    except Exception as e:
+        logger.error(f"Prediction error: {e}")
+        prediction = f"❌ An error occurred during prediction: {e}"
+
+    return prediction
+
+# Define Gradio interface
+def create_interface():
+    with gr.Blocks(css="#title {font-size: 2em; color: #4CAF50}") as demo:
+        gr.Markdown("# 🧠 Lip Reading App")
+        gr.Markdown("""This application allows you to perform lip reading by either uploading a video or recording directly using your webcam.""")
+
+        with gr.TabItem("Upload Video"):
+            with gr.Column():
+                video_input = gr.Video(
+                    label="📂 Upload Your Video",
+                    sources="upload"  # Specify source as upload
+                )
+                predict_button = gr.Button("🔍 Run Prediction")
+                prediction_output = gr.Textbox(
+                    label="📝 Predicted Text",
+                    interactive=False,
+                    lines=4,
+                    placeholder="Prediction will appear here."
+                )
+        
+        with gr.TabItem("Record Video"):
+            with gr.Column():
+                video_recorder = gr.Video(
+                    label="🎥 Record Your Video",
+                    sources="webcam"  # Specify source as webcam
+                )
+                predict_button_rec = gr.Button("🔍 Run Prediction on Recorded Video")
+                prediction_output_rec = gr.Textbox(
+                    label="📝 Predicted Text",
+                    interactive=False,
+                    lines=4,
+                    placeholder="Prediction will appear here."
+                )
+
+        # Add user instructions and feedback
+        with gr.Accordion("ℹ️ How to Use", open=False):
+            gr.Markdown("""
+            **Upload Video:**
+            - Click on the "Upload Your Video" button to select a video file from your device.
+            - Supported formats: MP4, AVI, MOV, MPG.
+            - After uploading, click "Run Prediction" to get the lip reading result.
+
+            **Record Video:**
+            - Click on the "Record Your Video" button to access your webcam.
+            - Grant the necessary permissions if prompted.
+            - Record your video and click "Stop Recording" (or equivalent) once done.
+            - Wait for 10 seconds until the recorded video appear on screen.
+            - Click "Run Prediction on Recorded Video" to get the lip reading result.
+            """)
+
+        # Define button actions
+        predict_button.click(fn=run_prediction, inputs=video_input, outputs=prediction_output)
+        predict_button_rec.click(fn=run_prediction, inputs=video_recorder, outputs=prediction_output_rec)
+
+        # Add footer or additional information if needed
+        gr.Markdown("""--- © 2024 Lip Reading App. All rights reserved.""")
+
+    return demo
+
+# Launch the interface
+if __name__ == "__main__":
+    demo = create_interface()
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=True  # Set to False if not sharing publicly
+    )

inference.py

@@ -0,0 +1,98 @@
+# inference3.py
+
+import tensorflow as tf
+from preprocessing import VideoPreprocessor
+
+class LipReadingModel:
+    def __init__(self, model_path='Lipnet.h5'):
+        # Initialize character mappings before loading the model
+        vocab = [x for x in "abcdefghijklmnopqrstuvwxyz'?!123456789 "]
+        self.char_to_num = tf.keras.layers.StringLookup(vocabulary=vocab, oov_token="")
+        self.num_to_char = tf.keras.layers.StringLookup(
+            vocabulary=self.char_to_num.get_vocabulary(), oov_token="", invert=True)
+
+        self.model = self.load_model()
+        try:
+            self.model.load_weights(model_path)
+            print("Model loaded successfully.")
+        except Exception as e:
+            print(f"Error loading model weights: {e}")
+
+    def load_model(self):
+        model = tf.keras.Sequential()
+        model.add(tf.keras.layers.Conv3D(128, (3, 3, 3), input_shape=(75, 75, 75, 1), padding='same'))
+        model.add(tf.keras.layers.Activation('relu'))
+        model.add(tf.keras.layers.MaxPooling3D((1, 2, 2)))
+
+        model.add(tf.keras.layers.Conv3D(256, (3, 3, 3), padding='same'))
+        model.add(tf.keras.layers.Activation('relu'))
+        model.add(tf.keras.layers.MaxPooling3D((1, 2, 2)))
+
+        model.add(tf.keras.layers.Conv3D(75, (3, 3, 3), padding='same'))
+        model.add(tf.keras.layers.Activation('relu'))
+        model.add(tf.keras.layers.MaxPooling3D((1, 2, 2)))
+
+        # Flatten the output for the RNN
+        model.add(tf.keras.layers.TimeDistributed(tf.keras.layers.Flatten()))
+
+        # LSTM layers
+        model.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(128, return_sequences=True, kernel_initializer='Orthogonal')))
+        model.add(tf.keras.layers.Dropout(0.5))
+
+        model.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(128, return_sequences=True, kernel_initializer='Orthogonal')))
+        model.add(tf.keras.layers.Dropout(0.5))
+
+        # Output layer
+        model.add(tf.keras.layers.Dense(self.char_to_num.vocabulary_size() + 1, activation='softmax'))
+
+        return model
+
+    def predict(self, normalized_frames):
+        if normalized_frames is None or normalized_frames.shape[0] == 0:
+            return "No frames extracted from the video. Please ensure the video contains a clear view of the face and lips."
+
+        frames = tf.expand_dims(normalized_frames, axis=0)  # Add batch dimension
+        yhat = self.model.predict(frames, verbose=0)
+
+        input_length = [yhat.shape[1]]  # batch size of 1
+        decoded = tf.keras.backend.ctc_decode(yhat, input_length=input_length, greedy=True)[0][0].numpy()[0]
+
+        # Convert numerical predictions to characters
+        prediction = ''.join(
+            [
+                self.num_to_char(num).numpy().decode('utf-8')
+                for num in decoded
+                if num != -1
+            ]
+        )
+
+        return prediction
+
+def predict_from_video(video_path=None, frames=None, model=None):
+    """
+    Predicts the text from a video file or webcam frames using the provided model.
+
+    Args:
+        video_path (str, optional): Path to the video file. Defaults to None.
+        frames (List[np.ndarray], optional): List of frames from webcam. Defaults to None.
+        model (LipReadingModel, optional): An instance of the LipReadingModel. Defaults to None.
+
+    Returns:
+        str: Predicted text.
+    """
+    if model is None:
+        model = LipReadingModel()
+
+    preprocessor = VideoPreprocessor()
+
+    if video_path:
+        # Preprocess video from file
+        normalized_frames = preprocessor.preprocess_video(video_path)
+    elif frames is not None:
+        # Preprocess frames from webcam
+        normalized_frames = preprocessor.preprocess_frames(frames)
+    else:
+        return "No video or frames provided for prediction."
+
+    prediction = model.predict(normalized_frames)
+    return prediction

preprocessing.py

@@ -0,0 +1,96 @@
+# preprocessing.py
+
+import cv2
+import mediapipe as mp
+import tensorflow as tf
+
+class VideoPreprocessor:
+    def __init__(self):
+        self.mp_face_mesh = mp.solutions.face_mesh
+        # Indices for lip landmarks
+        self.UPPER_LIP_INDICES = [61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291]
+        self.LOWER_LIP_INDICES = [146, 91, 181, 84, 17, 314, 405, 321, 375, 291]
+        self.LIP_INDICES = self.UPPER_LIP_INDICES + self.LOWER_LIP_INDICES
+
+    def preprocess_video(self, video_path):
+        cap = cv2.VideoCapture(video_path)
+        frames = []
+
+        # Utilize mediapipe's GPU acceleration if available
+        with self.mp_face_mesh.FaceMesh(
+            static_image_mode=False,
+            max_num_faces=1,
+            refine_landmarks=True,
+            min_detection_confidence=0.5,
+            min_tracking_confidence=0.5
+        ) as face_mesh:
+            while cap.isOpened():
+                ret, frame = cap.read()
+                if not ret:
+                    break
+
+                # Convert the BGR image to RGB
+                rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+
+                # Process the frame and get the facial landmarks
+                results = face_mesh.process(rgb_frame)
+
+                if results.multi_face_landmarks:
+                    # Get the landmarks for the first face
+                    face_landmarks = results.multi_face_landmarks[0]
+
+                    try:
+                        # Extract lip landmarks
+                        lip_landmarks = [face_landmarks.landmark[i] for i in self.LIP_INDICES]
+
+                        # Extract bounding box around the lips
+                        h, w, _ = frame.shape
+                        x_coords = [int(landmark.x * w) for landmark in lip_landmarks]
+                        y_coords = [int(landmark.y * h) for landmark in lip_landmarks]
+
+                        x_min, x_max = max(0, min(x_coords)), min(w, max(x_coords))
+                        y_min, y_max = max(0, min(y_coords)), min(h, max(y_coords))
+
+                        if x_max > x_min and y_max > y_min:
+                            # Crop the lip region
+                            lip_frame = frame[y_min:y_max, x_min:x_max]
+
+                            # Resize to 160x160
+                            lip_frame_resized = cv2.resize(lip_frame, (75, 75))
+
+                            # Convert to grayscale using TensorFlow
+                            lip_frame_gray = tf.image.rgb_to_grayscale(lip_frame_resized)
+
+                            frames.append(lip_frame_gray)
+                    except Exception as e:
+                        print(f"Error processing frame: {e}")
+                        continue  # Skip this frame
+                else:
+                    print("No face landmarks detected in frame.")
+
+        cap.release()
+
+        if not frames:
+            print("No frames extracted during preprocessing.")
+            return None  # Return None to indicate failure
+
+        # Stack frames into a tensor
+        frames = tf.stack(frames)
+
+        # Adjust frames to match expected input length
+        desired_num_frames = 75
+        num_frames = frames.shape[0]
+        if num_frames < desired_num_frames:
+            # Pad frames with zeros
+            padding = tf.zeros((desired_num_frames - num_frames, 75, 75, 1), dtype=tf.float32)
+            frames = tf.concat([frames, padding], axis=0)
+        elif num_frames > desired_num_frames:
+            # Truncate frames to desired_num_frames
+            frames = frames[:desired_num_frames]
+
+        # Normalize the frames
+        mean = tf.math.reduce_mean(frames)
+        std = tf.math.reduce_std(tf.cast(frames, tf.float32))
+        normalized_frames = tf.cast((frames - mean), tf.float32) / std
+
+        return normalized_frames  # Return TensorFlow tensor

requirements.txt

@@ -0,0 +1,9 @@
+tensorflow
+numpy
+imageio
+moviepy
+mediapipe
+opencv-python
+keras
+matplotlib
+gradio