Update app.py

app.py

@@ -1,24 +1,341 @@
-import streamlit as st
 import cv2
+import mediapipe as mp
 import numpy as np
+import time
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import Dense, Conv1D, MaxPooling1D, Flatten, Dropout
+from tensorflow.keras.optimizers import Adam
+# VECTORIZATION the u factor
+from scipy.signal import convolve2d
+import matplotlib.pyplot as plt
+import math
+from skimage import io, color
+from tensorflow import keras
+from tensorflow.keras.models import load_model
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.metrics import accuracy_score
+import os
+import torch
+import torchvision.transforms as transforms
+import torchvision.models as models
+import joblib
+import h5py
+from PIL import Image
+import streamlit as st
+
+
+
+
+
+# Load the model
+model = load_model('best_model_HQ_v8.h5')
+
+num_bins = 256
+
+def u_sliding_factor(image_channel, P):
+
+    result = np.zeros(image_channel.shape, np.float32)
+
+    # Define the sliding window size
+
+    window_size = (3, 3)
+
+    # Create the convolution kernel
+    kernel = np.ones(window_size, np.float32)
+    kernel[1, 1] = 0
+    kernel = kernel / (2 * P)
+    kernal2 = np.zeros(window_size, np.float32)
+    kernal2[1, 1] = 1
+    kernal2 = kernal2 / 2
+    # Perform the convolution using scipy's convolve2d
+    convolution_matrix = cv2.filter2D(image_channel,-1,kernel) + cv2.filter2D(image_channel,-1,kernal2)
+    result = convolution_matrix[1:-1, 1:-1]
+    
+    return result.astype(np.float32)
+
+    # Compute the C factor in this function which return a list of 8 values
+
+
+    # calculate C factor
+
+def C_list_calculate(P):
+    C = []
+    for count in range(1, 9):
+        c_value = ((P - count) * (count - 1)) / math.floor(((P - 1) / 2)**2)
+        C.append(c_value)
+    return C
+
+
+# The equilibriam Local Binary Pattern to compute the Texture values of an image
 
-def detect_face(image):
-    face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
-    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
-    faces = face_cascade.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=5, minSize=(30, 30))
-    for (x, y, w, h) in faces:
-        cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
-    return image
+def ED_LBP_Sliding_Matrix(I, P):
+    # Define the amount of padding
+    padding_amount = 1
 
-st.title("Face Detection App")
+    # Pad the array with zeros
+    I = np.pad(I, pad_width=padding_amount, mode='constant')
+    K = (2**P) - 1
+    C_list = C_list_calculate(8)
+    u_fac_matrix = u_sliding_factor(I.astype(np.float32),P)
+    slid_factor = np.zeros((u_fac_matrix.shape),np.float32)
+    m, n = u_fac_matrix.shape
+    ED_LBP = np.zeros(u_fac_matrix.shape,np.float32)
+    ED_LBP_matrix = np.zeros((u_fac_matrix.shape),np.float32)
+    K_matrix = np.ones(u_fac_matrix.shape).astype(np.float32) * K
+    offsets = [(0, 1), (0, 2), (1, 2), (2, 2), (2, 1), (2, 0), (1, 0),(0,0)]
+    count = 1
+    for offset in offsets:
+        row_offset, col_offset = offset
+        sliding_matrix = I[row_offset:row_offset+m, col_offset:col_offset+n].astype(np.float32) - u_fac_matrix.astype(np.float32)
+        slid_factor = np.maximum(sliding_matrix,0).astype(np.float32)
+        k_norm =  K_matrix.astype(np.float32) - u_fac_matrix.astype(np.float32)
+        # Replace zero values with a small non-zero value to avoid division by zero
+        k_norm_nonzero = np.where(k_norm == 0, 1e-10, k_norm)
+    # Calculate A_factor with the corrected k_norm_nonzero
+        A_factor = np.where(k_norm != 0, slid_factor / k_norm_nonzero, 0)
+        #A_factor = np.where(k_norm != 0, slid_factor / k_norm, 0)
+        ED_LBP_matrix = (A_factor.astype(np.float32) * C_list[count-1]) + np.ones(A_factor.shape).astype(np.float32)
+        ED_LBP  = ED_LBP + np.where(sliding_matrix >= 0, 2**((count-1)*ED_LBP_matrix.astype(np.float32)),0)
+        count  = count + 1
+    ED_LBP = np.where(ED_LBP > 255, 255, np.round(ED_LBP))    
 
-uploaded_image = st.file_uploader("Upload an image", type=["jpg", "jpeg", "png"])
-if uploaded_image is not None:
-    image = cv2.imdecode(np.fromstring(uploaded_image.read(), np.uint8), 1)
-    st.image(image, caption="Uploaded Image", use_column_width=True)
+    return ED_LBP.astype(int)
+
+# Compute Hostogram for the images  to extract the Vector of Feachres
+
+
+def compute_histogram(image, num_bins):
+    hist = cv2.calcHist([image], [0], None, [num_bins], [0, num_bins])
+    hist = hist / hist.sum()  # Normalize the histogram
+    return hist
+
+def spatial_pyramid(image, num_bins):
+    ED_LBP_image = np.zeros((image.shape),np.int16)
+    num_channels = image.shape[2]
+    histograms = []
+
+    for channel in range(num_channels):
+        #convert image to rgp
+    
+        ED_LBP_image[:, :, channel] = ED_LBP_Sliding_Matrix(image[:, :, channel].astype(np.int16), 8)
+        
+        # compute ED_LBP image
+        
+        # Level 0: Compute histogram for the entire channel
+        H1_channel = compute_histogram(ED_LBP_image[:, :, channel].astype(np.uint8), num_bins).ravel()  # Flatten H1_channel # Take care
+        
+        # Level 2: Compute histograms for 4x4 grids
+        grid_size = 4
+        H2_channel = np.empty((grid_size, grid_size, num_bins))
+        grid_height, grid_width = ED_LBP_image[:, :, channel].shape[0] // grid_size, ED_LBP_image[:, :, channel].shape[1] // grid_size
+        for m in range(grid_size):
+            for n in range(grid_size):
+                grid_image = ED_LBP_image[m * grid_height : (m + 1) * grid_height,
+                                n * grid_width : (n + 1) * grid_width, channel]
+                H2_channel[m, n] = compute_histogram(grid_image.astype(np.uint8), num_bins).ravel()  # Flatten the histogram
+        
+        H2_channel = H2_channel.reshape(-1)  # Flatten H2_channel
+        
+        # Concatenate histograms fr.om level 0 and level 2
+        Hs_channel = np.concatenate((H1_channel, H2_channel))
+        histograms.append(Hs_channel)
+    
+    # Concatenate histograms from all channels
+    feature_vector = np.concatenate(histograms)
+    return feature_vector
+
+
+# Land marks Detection 
+# Initialize variables to control frame capture interval
+capture_interval = 10  # Capture one frame every 10 seconds
+last_capture_time = 0
+
+mp_face_mesh = mp.solutions.face_mesh
+face_mesh = mp_face_mesh.FaceMesh(min_detection_confidence=0.5, min_tracking_confidence=0.5)
+mp_drawing = mp.solutions.drawing_utils
+drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
+cap = cv2.VideoCapture(0)
+# Streamlit
+st.title("Liveness Detector")
+frame_place_holder = st.empty()
+text_element = st.empty()
+stop_button_pressed = st.button("Stop")
+# Initialize flags to track face detection and image saving
+valid_face_detected = False
+image_saved = False
+
+while cap.isOpened() and not stop_button_pressed:
+    success, image = cap.read()
+    if not success:
+        st.write("End video")
+        break
+    start = time.time()
+    save_first_frame = True
+    #image = cv2.resize(image, (640, 480))
+    rotated_image = cv2.transpose(image)
+    #image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+    image.flags.writeable = False
+    results = face_mesh.process(image)
+    image.flags.writeable = True
+    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+    img_h, img_w, img_c = image.shape
+    face_3d = []
+    face_2d = []
+    if results.multi_face_landmarks:
+        for face_landmarks in results.multi_face_landmarks:
+            for idx, lm in enumerate(face_landmarks.landmark):
+                if idx==33 or idx==263 or idx==1 or idx==61 or idx==291 or idx==199:
+                    if idx == 1:
+                        nose_2d = (lm.x * img_w, lm.y * img_h)
+                        nose_3d = (lm.x * img_w, lm.y * img_h, lm.z * 3000)
+                    x, y = int(lm.x * img_w), int(lm.y * img_h)
+                    
+                    # Get the 2d cordinate
+                    face_2d.append([x, y])
+                    
+                    # Get 3d cordinate
+                    
+                    face_3d.append([x, y, lm.z])
+            #convert to numpy array
+            face_2d = np.array(face_2d, dtype=np.float32)
+            face_3d = np.array(face_3d, dtype=np.float32)
+            
+            #The camera matrix
+            
+            focal_length = 1 * img_w
+            #The distance matrix
+            cam_matrix = np.array([[focal_length, 0, img_h / 2],
+                                   [0, focal_length, img_w / 2],
+                                   [0, 0, 1]])
+            # the distance matrix
+            dist_matrix = np.zeros((4, 1), dtype=np.float64)
+            #solve PnP
+            success, rot_vec, trans_vec = cv2.solvePnP(face_3d, face_2d, cam_matrix, dist_matrix)
+            
+            #get rotational matrix
+            rmat ,jac = cv2.Rodrigues(rot_vec)
+            
+            #Get angles1
+            angles, mtxR, mtxQ, Qx, Qy, Qz = cv2.RQDecomp3x3(rmat)
+            
+            #get y rotation degree
+            x = angles[0] * 360
+            y = angles[1] * 360
+            z = angles[2] * 360
+            #text = "No Face"
+            # see where the user's head tilting
+            if y < -10:
+                text = "Look Right"
+            elif y > 10:
+                text = "Look Left"
+            elif x < -10:
+                text = "Look Up"
+            elif x > 10:
+                text = "Look Down"
+            else:
+                #text = "Forward"
+                current_time = time.time()
+                if current_time - last_capture_time >= capture_interval:
+                    last_capture_time = current_time
+                    image_data = []
+                    features_list=[]
+                    features_list2=[]
+                    # Check if there are face landmarks detected
+                    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+
+
+                    # Detect faces using cascade classifier
+                    face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
+                    faces = face_cascade.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=5, minSize=(30, 30))
+                    expansion_factor = 1.5
+                    num_bins = 256
+                    biggest_face = None
+                    biggest_area = 0
+                    target_size = (512,512)
+                    for (x, y, w, h) in faces:
+                        # Calculate the expanded dimensions
+                        expanded_x = max(0, int(x - (w * (expansion_factor - 1) / 2)))
+                        expanded_y = max(0, int(y - (h * (expansion_factor - 1) / 2)))
+                        expanded_w = min(img_w, int(w * expansion_factor))
+                        expanded_h = min(img_h, int(h * expansion_factor))
+
+                        # Crop the expanded face region from the frame
+                        current_area = expanded_w * expanded_h
+                        if current_area > biggest_area:
+                            biggest_area = current_area
+                            # biggest_face = image[expanded_y:expanded_y + expanded_h, expanded_x:expanded_x + expanded_w]
+                            biggest_face = image[y:y + h, x:x + w]
+                            resized_face = cv2.resize(biggest_face, target_size)
+                            resized_face = cv2.cvtColor(resized_face, cv2.COLOR_BGR2RGB)
+                            if save_first_frame:
+                                # Save the first frame as a JPG image
+                                cv2.imwrite('first_frame.jpg', resized_face)
+                                save_first_frame = False  # Disable savin
+                    if biggest_face is not None:
+                            
+                        # Perform spatial pyramid feature extraction
+                        rgb_features = spatial_pyramid(cv2.cvtColor(resized_face, cv2.COLOR_BGR2RGB), num_bins)
+                        hsv_features = spatial_pyramid(cv2.cvtColor(resized_face, cv2.COLOR_BGR2HSV), num_bins)
+                        ycbcr_features = spatial_pyramid(cv2.cvtColor(resized_face, cv2.COLOR_BGR2YCrCb), num_bins)
+                        if rgb_features.size > 0 and hsv_features.size > 0 and ycbcr_features.size > 0:
+                            combined_features = np.concatenate((rgb_features, hsv_features, ycbcr_features))
+                            features_list.append(combined_features)
+                                                        
+                    if len(features_list) > 0:
+                        X_array = np.array(features_list)
+                        X_test_array_reshaped = np.expand_dims(X_array, axis=-1)
+                        prediction = model.predict(X_test_array_reshaped)
+
+
+                        if prediction >= 0.74:
+                            text2 = str(prediction[0])
+                            valid_face_detected = True
+                            if valid_face_detected and not image_saved:
+                                # Save the current face image
+                                cv2.imwrite('current_face.jpg', resized_face)
+                                image_saved = True  # Set this flag to indicate the image has been saved
+
+                            #save current resized_face
+                        elif prediction < 0.74:
+                            text= "Not Live Image"
+                            text2 = str(prediction[0]) 
+                        else:
+                            text = "Fake Image"
+     
+            # Display the nose direction
+            nose_3d_projection, jacobian = cv2.projectPoints(nose_3d, rot_vec, trans_vec, cam_matrix, dist_matrix, dist_matrix)
+            
+            p1 = (int(nose_2d[0]), int(nose_2d[1]))
+            p2 = (int(nose_2d[0] + y*10), int(nose_2d[1] - x * 10))
+            
+            cv2.line(image, p1, p2, (255,0,0), 3)
+            cv2.putText(image, text, (20, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 0, 0))
+            cv2.putText(image, "x :" + str(np.round(x,2)), (500, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2)
+            cv2.putText(image, "y :" + str(np.round(x,2)), (500, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2)
+            cv2.putText(image, "z :" + str(np.round(x,2)), (500, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2)
+            mp_drawing.draw_landmarks(
+                image=image,
+                landmark_list=face_landmarks,
+                connections=mp_face_mesh.FACEMESH_TESSELATION,
+                landmark_drawing_spec=drawing_spec,
+                connection_drawing_spec=drawing_spec)
+    else:
+        text = "There is no Face"             
+        #Add the text on the image
+        cv2.putText(image, text, (20, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 0, 0))
+
+    end = time.time()
+    totalTime = end - start
     
-    if st.button("Detect Faces"):
-        result_image = detect_face(image)
-        st.image(result_image, caption="Image with Detected Faces", use_column_width=True)
+    fps = 1 / totalTime
+
+    cv2.putText(image, f'FPS: {int(fps)}', (20,450), cv2.FONT_HERSHEY_SIMPLEX, 1.5, (255, 0, 0), 2)
+
+    frame_place_holder.image(cv2.cvtColor(image, cv2.COLOR_BGR2RGB), use_column_width=True, channels="BGR")
 
-st.write("This is a simple face detection app using Streamlit and OpenCV.")
+    if (cv2.waitKey(5) & 0xff ==27) or stop_button_pressed:
+        break
+    valid_face_detected = False
+    image_saved = False    
+                
+cap.release()