Upload 12 files

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+best_model_sequences.keras filter=lfs diff=lfs merge=lfs -text
+final_model_sequences.keras filter=lfs diff=lfs merge=lfs -text
+shape_predictor_68_face_landmarks.dat filter=lfs diff=lfs merge=lfs -text

best_model_sequences.keras

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80f527be8dbdcfb36488d84cecad86368a1c8c4f618eafd204e6f23bcbef42ed
+size 342613105

data_preprocessing_sequences.py

@@ -0,0 +1,174 @@
+# data_preprocessing_sequences.py
+
+import os
+import cv2
+import dlib
+import numpy as np
+from imutils import face_utils
+from tqdm import tqdm
+import pickle
+
+def get_facial_landmarks(detector, predictor, image):
+    """
+    Detects facial landmarks in an image.
+
+    Args:
+        detector: dlib face detector.
+        predictor: dlib shape predictor.
+        image (numpy.ndarray): Input image.
+
+    Returns:
+        dict: Coordinates of eyes and eyebrows.
+    """
+    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+    rects = detector(gray, 1)
+    
+    if len(rects) == 0:
+        return None  # No face detected
+    
+    # Assuming the first detected face is the target
+    rect = rects[0]
+    shape = predictor(gray, rect)
+    shape = face_utils.shape_to_np(shape)
+    
+    landmarks = {}
+    # Define landmarks for left and right eyes and eyebrows
+    landmarks['left_eye'] = shape[36:42]      # Left eye landmarks
+    landmarks['right_eye'] = shape[42:48]     # Right eye landmarks
+    landmarks['left_eyebrow'] = shape[17:22]  # Left eyebrow landmarks
+    landmarks['right_eyebrow'] = shape[22:27] # Right eyebrow landmarks
+    
+    return landmarks
+
+def extract_roi(image, landmarks, region='left_eye', padding=5):
+    """
+    Extracts a region of interest (ROI) from the image based on landmarks.
+
+    Args:
+        image (numpy.ndarray): Input image.
+        landmarks (dict): Facial landmarks.
+        region (str): Region to extract ('left_eye', 'right_eye', 'left_eyebrow', 'right_eyebrow').
+        padding (int): Padding around the ROI.
+
+    Returns:
+        numpy.ndarray: Extracted ROI.
+    """
+    points = landmarks.get(region)
+    if points is None:
+        return None
+    
+    # Compute the bounding box
+    x, y, w, h = cv2.boundingRect(points)
+    x = max(x - padding, 0)
+    y = max(y - padding, 0)
+    w = w + 2 * padding
+    h = h + 2 * padding
+    
+    roi = image[y:y+h, x:x+w]
+    return roi
+
+def preprocess_video_sequence(sequence_dir, detector, predictor, img_size=(64, 64)):
+    """
+    Preprocesses a sequence of frames from a video.
+
+    Args:
+        sequence_dir (str): Directory containing frames of a video.
+        detector: dlib face detector.
+        predictor: dlib shape predictor.
+        img_size (tuple): Desired image size for ROIs.
+
+    Returns:
+        list: List of preprocessed frames as numpy arrays.
+    """
+    frames = sorted([f for f in os.listdir(sequence_dir) if f.endswith('.jpg') or f.endswith('.png')])
+    preprocessed_sequence = []
+    
+    for frame_name in frames:
+        frame_path = os.path.join(sequence_dir, frame_name)
+        image = cv2.imread(frame_path)
+        if image is None:
+            continue
+        
+        landmarks = get_facial_landmarks(detector, predictor, image)
+        if landmarks is None:
+            continue  # Skip frames with no detected face
+        
+        # Extract ROIs for eyes and eyebrows
+        rois = {}
+        rois['left_eye'] = extract_roi(image, landmarks, 'left_eye')
+        rois['right_eye'] = extract_roi(image, landmarks, 'right_eye')
+        rois['left_eyebrow'] = extract_roi(image, landmarks, 'left_eyebrow')
+        rois['right_eyebrow'] = extract_roi(image, landmarks, 'right_eyebrow')
+        
+        # Process ROIs
+        roi_images = []
+        for region in ['left_eye', 'right_eye', 'left_eyebrow', 'right_eyebrow']:
+            roi = rois.get(region)
+            if roi is not None:
+                roi = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY)  # Convert to grayscale
+                roi = cv2.resize(roi, img_size)
+                roi = roi.astype('float32') / 255.0        # Normalize to [0,1]
+                roi = np.expand_dims(roi, axis=-1)          # Add channel dimension
+                roi_images.append(roi)
+        
+        if len(roi_images) == 0:
+            continue  # Skip if no ROIs were extracted
+        
+        # Concatenate ROIs horizontally to form a single image
+        combined_roi = np.hstack(roi_images)
+        preprocessed_sequence.append(combined_roi)
+    
+    return preprocessed_sequence
+
+def preprocess_dataset(dataset_dir='dataset', output_dir='preprocessed_sequences', img_size=(64, 64)):
+    """
+    Preprocesses the entire dataset by processing each video sequence.
+
+    Args:
+        dataset_dir (str): Directory containing labeled data.
+        output_dir (str): Directory to save preprocessed sequences.
+        img_size (tuple): Desired image size for ROIs.
+    """
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+    
+    # Initialize dlib's face detector and landmark predictor
+    detector = dlib.get_frontal_face_detector()
+    predictor_path = 'shape_predictor_68_face_landmarks.dat'
+    
+    if not os.path.exists(predictor_path):
+        print(f"Error: {predictor_path} not found. Download it from http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2")
+        return
+    
+    predictor = dlib.shape_predictor(predictor_path)
+    
+    classes = os.listdir(dataset_dir)
+    for cls in classes:
+        cls_path = os.path.join(dataset_dir, cls)
+        if not os.path.isdir(cls_path):
+            continue
+        output_cls_dir = os.path.join(output_dir, cls)
+        if not os.path.exists(output_cls_dir):
+            os.makedirs(output_cls_dir)
+        
+        print(f"Processing class: {cls}")
+        sequences = os.listdir(cls_path)
+        for seq in tqdm(sequences, desc=f"Class {cls}"):
+            seq_path = os.path.join(cls_path, seq)
+            if not os.path.isdir(seq_path):
+                continue
+            preprocessed_sequence = preprocess_video_sequence(seq_path, detector, predictor, img_size=img_size)
+            if len(preprocessed_sequence) == 0:
+                continue  # Skip sequences with no valid frames
+            
+            # Stack frames to form a 3D array (frames, height, width, channels)
+            sequence_array = np.stack(preprocessed_sequence, axis=0)
+            
+            # Save the preprocessed sequence as a numpy file
+            npy_filename = os.path.join(output_cls_dir, f"{seq}.npy")
+            np.save(npy_filename, sequence_array)
+    
+    print("Data preprocessing completed.")
+
+if __name__ == "__main__":
+    preprocess_dataset(dataset_dir='dataset', output_dir='preprocessed_sequences', img_size=(64, 64))

dataset_preparation_sequences.py

@@ -0,0 +1,116 @@
+# dataset_preparation_sequences.py
+
+import os
+import numpy as np
+from sklearn.model_selection import train_test_split
+from tensorflow.keras.utils import to_categorical
+import pickle
+
+def load_sequences(preprocessed_dir='preprocessed_sequences'):
+    """
+    Loads preprocessed sequences and their labels.
+
+    Args:
+        preprocessed_dir (str): Directory containing preprocessed sequences.
+
+    Returns:
+        tuple: Lists of sequences and labels, label mapping dictionary.
+    """
+    X = []
+    y = []
+    label_map = {}
+    classes = sorted(os.listdir(preprocessed_dir))
+    
+    for idx, cls in enumerate(classes):
+        label_map[cls] = idx
+        cls_path = os.path.join(preprocessed_dir, cls)
+        if not os.path.isdir(cls_path):
+            continue
+        sequence_files = [f for f in os.listdir(cls_path) if f.endswith('.npy')]
+        for seq_file in sequence_files:
+            seq_path = os.path.join(cls_path, seq_file)
+            sequence = np.load(seq_path)
+            X.append(sequence)
+            y.append(idx)
+    
+    # X remains a list of numpy arrays with varying shapes
+    y = np.array(y)
+    y = to_categorical(y, num_classes=len(label_map))
+    
+    return X, y, label_map
+
+def pad_sequences_fixed(X, max_seq_length):
+    """
+    Pads or truncates sequences to a fixed length.
+
+    Args:
+        X (list of numpy.ndarray): List of sequences with shape (frames, height, width, channels).
+        max_seq_length (int): Desired sequence length.
+
+    Returns:
+        numpy.ndarray: Padded/truncated sequences.
+    """
+    padded_X = []
+    for seq in X:
+        if seq.shape[0] < max_seq_length:
+            pad_width = max_seq_length - seq.shape[0]
+            padding = np.zeros((pad_width, *seq.shape[1:]), dtype=seq.dtype)
+            padded_seq = np.concatenate((seq, padding), axis=0)
+        else:
+            padded_seq = seq[:max_seq_length]
+        padded_X.append(padded_seq)
+    return np.array(padded_X)
+
+def save_dataset(X_train, X_test, y_train, y_test, label_map, output_path='dataset_sequences.pkl'):
+    """
+    Saves the dataset into a pickle file.
+
+    Args:
+        X_train, X_test, y_train, y_test: Split data.
+        label_map (dict): Mapping from class names to indices.
+        output_path (str): Path to save the pickle file.
+    """
+    with open(output_path, 'wb') as f:
+        pickle.dump({
+            'X_train': X_train,
+            'X_test': X_test,
+            'y_train': y_train,
+            'y_test': y_test,
+            'label_map': label_map
+        }, f)
+    print(f"Dataset saved to {output_path}.")
+
+def load_dataset_pickle(pickle_path='dataset_sequences.pkl'):
+    """
+    Loads the dataset from a pickle file.
+
+    Args:
+        pickle_path (str): Path to the pickle file.
+
+    Returns:
+        tuple: Split data and label mapping.
+    """
+    with open(pickle_path, 'rb') as f:
+        data = pickle.load(f)
+    return data['X_train'], data['X_test'], data['y_train'], data['y_test'], data['label_map']
+
+if __name__ == "__main__":
+    # Load sequences
+    X, y, label_map = load_sequences(preprocessed_dir='preprocessed_sequences')
+    print(f"Total samples: {len(X)}")
+    
+    # Find the maximum sequence length for padding
+    max_seq_length = max([seq.shape[0] for seq in X])
+    print(f"Maximum sequence length: {max_seq_length}")
+    
+    # Pad sequences to have the same length
+    X_padded = pad_sequences_fixed(X, max_seq_length)
+    print(f"Padded sequences shape: {X_padded.shape}")
+    
+    # Split into training and testing sets
+    X_train, X_test, y_train, y_test = train_test_split(X_padded, y, test_size=0.2, random_state=42)
+    print(f"Training samples: {X_train.shape[0]}")
+    print(f"Testing samples: {X_test.shape[0]}")
+    
+    # Save the dataset
+    save_dataset(X_train, X_test, y_train, y_test, label_map, output_path='dataset_sequences.pkl')

dataset_sequences.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abbe955e05fb92417fc7bc27e88998dcf0c311b06bc0803d47483b9d844893e7
+size 196612009

final_model_sequences.keras

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:757fd4422e6703b3a4d9730edaee6643be3e1e32fefe71520d3b2bb341e62be3
+size 342613105

frame_extraction.py

@@ -0,0 +1,61 @@
+# frame_extraction.py
+
+import cv2
+import os
+from tqdm import tqdm
+
+def extract_frames(video_path, output_dir='frames', prefix='frame'):
+    """
+    Extracts frames from a video file.
+
+    Args:
+        video_path (str): Path to the input video file.
+        output_dir (str): Directory to save extracted frames.
+        prefix (str): Prefix for the frame filenames.
+    """
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+    
+    cap = cv2.VideoCapture(video_path)
+    
+    if not cap.isOpened():
+        print(f"Error: Could not open video {video_path}.")
+        return
+    
+    frame_count = 0
+    frame_total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+    
+    for _ in tqdm(range(frame_total), desc=f"Extracting frames from {os.path.basename(video_path)}"):
+        ret, frame = cap.read()
+        if not ret:
+            break
+        frame_filename = os.path.join(output_dir, f"{prefix}_frame_{frame_count}.jpg")
+        cv2.imwrite(frame_filename, frame)  # Save frame as JPEG file
+        frame_count += 1
+    
+    cap.release()
+    print(f"Extracted {frame_count} frames from {video_path}.")
+
+def process_all_videos(videos_dir='videos', frames_dir='frames'):
+    """
+    Processes all videos in the specified directory and extracts frames.
+
+    Args:
+        videos_dir (str): Directory containing video files.
+        frames_dir (str): Directory to save extracted frames.
+    """
+    if not os.path.exists(frames_dir):
+        os.makedirs(frames_dir)
+    
+    video_files = [f for f in os.listdir(videos_dir) if f.endswith('.avi') or f.endswith('.mp4')]
+    
+    for video_file in video_files:
+        label = video_file.split('_')[0]  # Assuming filename format 'label_something.avi'
+        video_path = os.path.join(videos_dir, video_file)
+        output_subdir = os.path.join(frames_dir, label)
+        if not os.path.exists(output_subdir):
+            os.makedirs(output_subdir)
+        extract_frames(video_path, output_dir=output_subdir, prefix=video_file.split('.')[0])
+
+if __name__ == "__main__":
+    process_all_videos(videos_dir='videos', frames_dir='frames')

history_sequences.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb39ac8c6291f9fccb70c39c7c9eab41d6c3499fe3cccdfcc88c51231fd07440
+size 1876

model_building_sequences.py

@@ -0,0 +1,110 @@
+# model_building_sequences.py
+
+import tensorflow as tf
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, TimeDistributed, LSTM, BatchNormalization
+from tensorflow.keras.optimizers import Adam
+from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping
+import pickle
+
+def build_cnn_lstm_model(input_shape, num_classes):
+    """
+    Builds a CNN-LSTM model for sequence classification.
+
+    Args:
+        input_shape (tuple): Shape of input sequences (frames, height, width, channels).
+        num_classes (int): Number of output classes.
+
+    Returns:
+        tensorflow.keras.Model: Compiled model.
+    """
+    model = Sequential()
+    
+    # Apply Conv2D to each frame in the sequence
+    model.add(TimeDistributed(Conv2D(32, (3,3), activation='relu'), input_shape=input_shape))
+    model.add(TimeDistributed(MaxPooling2D((2,2))))
+    model.add(TimeDistributed(BatchNormalization()))
+    
+    # Additional Conv2D layers
+    model.add(TimeDistributed(Conv2D(64, (3,3), activation='relu')))
+    model.add(TimeDistributed(MaxPooling2D((2,2))))
+    model.add(TimeDistributed(BatchNormalization()))
+    
+    # Flatten the output from Conv layers
+    model.add(TimeDistributed(Flatten()))
+    
+    # LSTM layer to capture temporal dependencies
+    model.add(LSTM(128, return_sequences=False))
+    model.add(Dropout(0.5))
+    
+    # Fully connected layer
+    model.add(Dense(128, activation='relu'))
+    model.add(Dropout(0.5))
+    
+    # Output layer with softmax activation for classification
+    model.add(Dense(num_classes, activation='softmax'))
+    
+    # Compile the model with Adam optimizer and categorical cross-entropy loss
+    model.compile(optimizer=Adam(learning_rate=1e-4),
+                  loss='categorical_crossentropy',
+                  metrics=['accuracy'])
+    
+    return model
+
+def load_dataset_pickle(pickle_path='dataset_sequences.pkl'):
+    """
+    Loads the dataset from a pickle file.
+
+    Args:
+        pickle_path (str): Path to the pickle file.
+
+    Returns:
+        tuple: Split data and label mapping.
+    """
+    with open(pickle_path, 'rb') as f:
+        data = pickle.load(f)
+    return data['X_train'], data['X_test'], data['y_train'], data['y_test'], data['label_map']
+
+def main():
+    # Load the dataset
+    X_train, X_test, y_train, y_test, label_map = load_dataset_pickle('dataset_sequences.pkl')
+    num_classes = y_train.shape[1]
+    input_shape = X_train.shape[1:]  # (frames, height, width, channels)
+    
+    # Build the CNN-LSTM model
+    model = build_cnn_lstm_model(input_shape, num_classes)
+    model.summary()
+    
+    # Define callbacks with updated filepath (.keras)
+    checkpoint = ModelCheckpoint(
+        'best_model_sequences.keras',  # Changed from .h5 to .keras
+        monitor='val_accuracy',
+        save_best_only=True,
+        mode='max'
+    )
+    early_stop = EarlyStopping(
+        monitor='val_accuracy',
+        patience=10,
+        restore_best_weights=True
+    )
+    
+    # Train the model using GPU
+    with tf.device('/GPU:0'):
+        history = model.fit(
+            X_train, y_train,
+            epochs=50,
+            batch_size=128,  # Adjust based on your system's memory
+            validation_data=(X_test, y_test)
+        )
+    
+    # Save the final trained model with .keras extension
+    model.save('final_model_sequences.keras')  # Changed from .h5 to .keras
+    print("Model training completed and saved as 'final_model_sequences.keras'.")
+    
+    # Save training history for future reference
+    with open('history_sequences.pkl', 'wb') as f:
+        pickle.dump(history.history, f)
+    print("Training history saved as 'history_sequences.pkl'.")
+
+if __name__ == "__main__":
+    main()

model_evaluation_sequences.py

@@ -0,0 +1,123 @@
+# model_evaluation_sequences.py
+
+import tensorflow as tf
+import pickle
+import matplotlib.pyplot as plt
+import numpy as np
+import seaborn as sns
+from sklearn.metrics import confusion_matrix, classification_report
+
+def load_model(model_path='best_model_sequences.keras'):
+    """
+    Loads the trained model.
+
+    Args:
+        model_path (str): Path to the saved model.
+
+    Returns:
+        tensorflow.keras.Model: Loaded model.
+    """
+    model = tf.keras.models.load_model(model_path)
+    return model
+
+def load_dataset_pickle(pickle_path='dataset_sequences.pkl'):
+    """
+    Loads the dataset from a pickle file.
+
+    Args:
+        pickle_path (str): Path to the pickle file.
+
+    Returns:
+        tuple: Split data and label mapping.
+    """
+    with open(pickle_path, 'rb') as f:
+        data = pickle.load(f)
+    return data['X_train'], data['X_test'], data['y_train'], data['y_test'], data['label_map']
+
+def plot_history(history):
+    """
+    Plots the training and validation accuracy and loss.
+
+    Args:
+        history (dict): Training history.
+    """
+    acc = history.get('accuracy', history.get('acc'))
+    val_acc = history.get('val_accuracy', history.get('val_acc'))
+    
+    loss = history['loss']
+    val_loss = history['val_loss']
+    
+    epochs = range(1, len(acc) + 1)
+    
+    plt.figure(figsize=(14,5))
+    
+    plt.subplot(1,2,1)
+    plt.plot(epochs, acc, 'b', label='Training accuracy')
+    plt.plot(epochs, val_acc, 'r', label='Validation accuracy')
+    plt.title('Training and Validation Accuracy')
+    plt.xlabel('Epochs')
+    plt.ylabel('Accuracy')
+    plt.legend()
+    
+    plt.subplot(1,2,2)
+    plt.plot(epochs, loss, 'b', label='Training loss')
+    plt.plot(epochs, val_loss, 'r', label='Validation loss')
+    plt.title('Training and Validation Loss')
+    plt.xlabel('Epochs')
+    plt.ylabel('Loss')
+    plt.legend()
+    
+    plt.show()
+
+def evaluate_model(model, X_test, y_test, label_map):
+    """
+    Evaluates the model on the test set.
+
+    Args:
+        model (tensorflow.keras.Model): Trained model.
+        X_test (numpy.ndarray): Test sequences.
+        y_test (numpy.ndarray): Test labels.
+        label_map (dict): Mapping from class names to indices.
+    """
+    loss, accuracy = model.evaluate(X_test, y_test, verbose=0)
+    print(f"Test Accuracy: {accuracy * 100:.2f}%")
+    print(f"Test Loss: {loss:.4f}")
+    
+    # Predictions
+    y_pred = model.predict(X_test)
+    y_pred_classes = np.argmax(y_pred, axis=1)
+    y_true = np.argmax(y_test, axis=1)
+    
+    # Confusion Matrix
+    cm = confusion_matrix(y_true, y_pred_classes)
+    plt.figure(figsize=(10,8))
+    sns.heatmap(cm, annot=True, fmt='d', xticklabels=label_map.keys(), yticklabels=label_map.keys(), cmap='Blues')
+    plt.xlabel('Predicted')
+    plt.ylabel('True')
+    plt.title('Confusion Matrix')
+    plt.show()
+    
+    # Classification Report
+    print("Classification Report:")
+    print(classification_report(y_true, y_pred_classes, target_names=label_map.keys()))
+
+def main():
+    # Load the trained model
+    model = load_model('best_model_sequences.keras')
+    
+    # Load the dataset
+    X_train, X_test, y_train, y_test, label_map = load_dataset_pickle('dataset_sequences.pkl')
+    
+    # Evaluate the model
+    evaluate_model(model, X_test, y_test, label_map)
+    
+    # Load and plot training history
+    try:
+        with open('history_sequences.pkl', 'rb') as f:
+            history = pickle.load(f)
+        plot_history(history)
+    except FileNotFoundError:
+        print("Training history not found. Skipping plotting.")
+
+if __name__ == "__main__":
+    main()

prediction_sequences.py

@@ -0,0 +1,284 @@
+# prediction_sequences.py
+
+import tensorflow as tf
+import cv2
+import numpy as np
+import dlib
+from imutils import face_utils
+import os
+import pickle
+from collections import deque
+import threading
+import queue
+import time
+
+def load_model(model_path='final_model_sequences.keras'):
+    """
+    Loads the trained model.
+
+    Args:
+        model_path (str): Path to the saved model.
+
+    Returns:
+        tensorflow.keras.Model: Loaded model.
+    """
+    model = tf.keras.models.load_model(model_path)
+    return model
+
+def get_facial_landmarks(detector, predictor, image):
+    """
+    Detects facial landmarks in an image.
+
+    Args:
+        detector: dlib face detector.
+        predictor: dlib shape predictor.
+        image (numpy.ndarray): Input image.
+
+    Returns:
+        dict: Coordinates of eyes and eyebrows.
+    """
+    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+    rects = detector(gray, 1)
+
+    if len(rects) == 0:
+        return None  # No face detected
+
+    # Assuming the first detected face is the target
+    rect = rects[0]
+    shape = predictor(gray, rect)
+    shape = face_utils.shape_to_np(shape)
+
+    landmarks = {}
+    # Define landmarks for left and right eyes and eyebrows
+    landmarks['left_eye'] = shape[36:42]      # Left eye landmarks
+    landmarks['right_eye'] = shape[42:48]     # Right eye landmarks
+    landmarks['left_eyebrow'] = shape[17:22]  # Left eyebrow landmarks
+    landmarks['right_eyebrow'] = shape[22:27] # Right eyebrow landmarks
+
+    return landmarks
+
+def extract_roi(image, landmarks, region='left_eye', padding=5):
+    """
+    Extracts a region of interest (ROI) from the image based on landmarks.
+
+    Args:
+        image (numpy.ndarray): Input image.
+        landmarks (dict): Facial landmarks.
+        region (str): Region to extract ('left_eye', 'right_eye', 'left_eyebrow', 'right_eyebrow').
+        padding (int): Padding around the ROI.
+
+    Returns:
+        numpy.ndarray: Extracted ROI.
+    """
+    points = landmarks.get(region)
+    if points is None:
+        return None
+
+    # Compute the bounding box
+    x, y, w, h = cv2.boundingRect(points)
+    x = max(x - padding, 0)
+    y = max(y - padding, 0)
+    w = w + 2 * padding
+    h = h + 2 * padding
+
+    roi = image[y:y+h, x:x+w]
+    return roi
+
+def preprocess_frame(image, detector, predictor, img_size=(64, 64)):
+    """
+    Preprocesses a single frame: detects landmarks, extracts ROIs, and prepares the input.
+
+    Args:
+        image (numpy.ndarray): Input frame.
+        detector: dlib face detector.
+        predictor: dlib shape predictor.
+        img_size (tuple): Desired image size for ROIs.
+
+    Returns:
+        numpy.ndarray: Preprocessed frame as a concatenated ROI image.
+    """
+    landmarks = get_facial_landmarks(detector, predictor, image)
+    if landmarks is None:
+        return None  # No face detected
+
+    # Extract ROIs for eyes and eyebrows
+    rois = {}
+    rois['left_eye'] = extract_roi(image, landmarks, 'left_eye')
+    rois['right_eye'] = extract_roi(image, landmarks, 'right_eye')
+    rois['left_eyebrow'] = extract_roi(image, landmarks, 'left_eyebrow')
+    rois['right_eyebrow'] = extract_roi(image, landmarks, 'right_eyebrow')
+
+    # Process ROIs
+    roi_images = []
+    for region in ['left_eye', 'right_eye', 'left_eyebrow', 'right_eyebrow']:
+        roi = rois.get(region)
+        if roi is not None:
+            roi = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY)  # Convert to grayscale
+            roi = cv2.resize(roi, img_size)
+            roi = roi.astype('float32') / 255.0        # Normalize to [0,1]
+            roi = np.expand_dims(roi, axis=-1)         # Add channel dimension
+            roi_images.append(roi)
+
+    if len(roi_images) == 0:
+        return None  # No ROIs extracted
+
+    # Concatenate ROIs horizontally to form a single image
+    combined_roi = np.hstack(roi_images)
+    return combined_roi
+
+def movement_to_text(label_map):
+    """
+    Creates a mapping from class indices to text.
+
+    Args:
+        label_map (dict): Mapping from class names to indices.
+
+    Returns:
+        dict: Mapping from indices to text descriptions.
+    """
+    movement_to_text_map = {
+        'upward_eyebrow': 'Eyebrow Raised',
+        'downward_eyebrow': 'Eyebrow Lowered',
+        'left_eye': 'Left Eye Movement',
+        'right_eye': 'Right Eye Movement',
+        # Add more mappings as needed
+    }
+
+    # Create index to text mapping
+    index_to_text = {}
+    for cls, idx in label_map.items():
+        text = movement_to_text_map.get(cls, cls)
+        index_to_text[idx] = text
+    return index_to_text
+
+def prediction_worker(model, input_queue, output_queue, max_seq_length):
+    """
+    Worker thread for handling model predictions.
+
+    Args:
+        model (tensorflow.keras.Model): Trained model.
+        input_queue (queue.Queue): Queue to receive sequences for prediction.
+        output_queue (queue.Queue): Queue to send prediction results.
+        max_seq_length (int): Fixed sequence length for the model.
+    """
+    while True:
+        sequence = input_queue.get()
+        if sequence is None:
+            break  # Sentinel to stop the thread
+
+        # Pad or truncate the sequence to match the model's expected input
+        if sequence.shape[0] < max_seq_length:
+            pad_width = max_seq_length - sequence.shape[0]
+            padding = np.zeros((pad_width, *sequence.shape[1:]), dtype=sequence.dtype)
+            sequence_padded = np.concatenate((sequence, padding), axis=0)
+        else:
+            sequence_padded = sequence[:max_seq_length]
+
+        # Expand dimensions to match model input (1, frames, height, width, channels)
+        sequence_padded = np.expand_dims(sequence_padded, axis=0)
+
+        # Perform prediction
+        prediction = model.predict(sequence_padded)
+        class_idx = np.argmax(prediction)
+        confidence = np.max(prediction)
+
+        # Put the result in the output queue
+        output_queue.put((class_idx, confidence))
+
+def main():
+    # Load the trained model
+    model = load_model('final_model_sequences.keras')
+
+    # Load label map
+    with open('dataset_sequences.pkl', 'rb') as f:
+        data = pickle.load(f)
+    label_map = data['label_map']
+    index_to_text = movement_to_text(label_map)
+
+    # Initialize dlib's face detector and landmark predictor
+    detector = dlib.get_frontal_face_detector()
+    predictor_path = 'shape_predictor_68_face_landmarks.dat'
+
+    if not os.path.exists(predictor_path):
+        print(f"Error: {predictor_path} not found. Download it from http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2")
+        return
+
+    predictor = dlib.shape_predictor(predictor_path)
+
+    # Initialize queues for communication between threads
+    input_queue = queue.Queue()
+    output_queue = queue.Queue()
+
+    # Define sequence length (number of frames)
+    max_seq_length = 20  # Adjust based on your training data
+
+    # Start the prediction worker thread
+    pred_thread = threading.Thread(target=prediction_worker, args=(model, input_queue, output_queue, max_seq_length))
+    pred_thread.daemon = True
+    pred_thread.start()
+
+    # Start video capture
+    cap = cv2.VideoCapture(0)
+
+    if not cap.isOpened():
+        print("Error: Could not open webcam.")
+        return
+
+    print("Starting real-time prediction. Press 'q' to quit.")
+
+    # Initialize a deque to store the sequence of preprocessed frames
+    frame_buffer = deque(maxlen=max_seq_length)
+
+    # Variable to store the latest prediction result
+    latest_prediction = "Initializing..."
+
+    while True:
+        ret, frame = cap.read()
+        if not ret:
+            print("Failed to grab frame.")
+            break
+
+        # Preprocess the current frame
+        preprocessed_frame = preprocess_frame(frame, detector, predictor, img_size=(64, 64))
+        if preprocessed_frame is not None:
+            frame_buffer.append(preprocessed_frame)
+        else:
+            # If no face detected, append a zero array to maintain sequence length
+            frame_buffer.append(np.zeros((64, 256, 1), dtype='float32'))
+
+        # If the buffer is full, send the sequence to the prediction thread
+        if len(frame_buffer) == max_seq_length:
+            # Convert deque to numpy array
+            sequence_array = np.array(frame_buffer)
+            input_queue.put(sequence_array)
+
+        # Check if there's a new prediction result
+        try:
+            while True:
+                class_idx, confidence = output_queue.get_nowait()
+                movement = index_to_text.get(class_idx, "Unknown")
+                latest_prediction = f"{movement} ({confidence*100:.2f}%)"
+        except queue.Empty:
+            pass  # No new prediction
+
+        # Display the prediction on the frame
+        cv2.putText(frame, latest_prediction, (30, 30), cv2.FONT_HERSHEY_SIMPLEX, 
+                    0.8, (0, 255, 0), 2, cv2.LINE_AA)
+
+        # Display the frame
+        cv2.imshow('Real-time Movement Prediction', frame)
+
+        # Exit condition
+        if cv2.waitKey(1) & 0xFF == ord('q'):
+            break
+
+    # Cleanup
+    cap.release()
+    cv2.destroyAllWindows()
+
+    # Stop the prediction thread
+    input_queue.put(None)  # Sentinel to stop the thread
+    pred_thread.join()
+
+if __name__ == "__main__":
+    main()

shape_predictor_68_face_landmarks.dat

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbdc2cb80eb9aa7a758672cbfdda32ba6300efe9b6e6c7a299ff7e736b11b92f
+size 99693937

video_capture.py

@@ -0,0 +1,62 @@
+# video_capture.py
+
+import cv2
+import os
+
+def record_video(duration=2, output_dir='videos', filename='sample'):
+    """
+    Records a short video from the webcam.
+
+    Args:
+        duration (int): Duration of the video in seconds.
+        output_dir (str): Directory to save the videos.
+        filename (str): Name of the output video file.
+    """
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+    
+    cap = cv2.VideoCapture(0)  # Initialize webcam
+    
+    if not cap.isOpened():
+        print("Error: Could not open webcam.")
+        return
+    
+    # Get default camera resolution
+    frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    fps = 20  # Frames per second
+    
+    # Define the codec and create VideoWriter object
+    fourcc = cv2.VideoWriter_fourcc(*'XVID')
+    out = cv2.VideoWriter(os.path.join(output_dir, f"{filename}.avi"), fourcc, fps, (frame_width, frame_height))
+    
+    print("Recording started. Press 'q' to stop early.")
+    
+    frame_count = 0
+    total_frames = duration * fps
+    
+    while frame_count < total_frames:
+        ret, frame = cap.read()
+        if ret:
+            out.write(frame)  # Write frame to video file
+            cv2.imshow('Recording', frame)
+            frame_count += 1
+            
+            # Press 'q' to quit early
+            if cv2.waitKey(1) & 0xFF == ord('q'):
+                break
+        else:
+            print("Failed to grab frame.")
+            break
+    
+    # Release resources
+    cap.release()
+    out.release()
+    cv2.destroyAllWindows()
+    print(f"Recording finished. Video saved as {filename}.avi")
+
+if __name__ == "__main__":
+    # Example: Record a 2-second video named 'movement1'
+    label = input("Enter movement label (e.g., 'upward_eyebrow'): ")
+    filename = input("Enter filename (e.g., 'movement1'): ")
+    record_video(duration=2, filename=filename)