Update app.py

app.py

@@ -1,12 +1,139 @@
-import streamlit as st
-import numpy as np
+from turn import get_ice_servers
+
 import cv2
-from io import BytesIO
+import mediapipe as mp
+import numpy as np
+import time
+import math
+import streamlit as st
+import av
+
+from tensorflow.keras.models import load_model
+from scipy.signal import convolve2d
+from skimage import color
+from skimage import io
+from sklearn.metrics import accuracy_score
+
+# VECTORIZATION the u factor
+import matplotlib.pyplot as plt
+import os
+import torch
+import torchvision.transforms as transforms
+import torchvision.models as models
+from PIL import Image
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import Dense, Conv1D, MaxPooling1D, Flatten, Dropout
+from tensorflow.keras.optimizers import Adam
+from streamlit_webrtc import webrtc_streamer
+
+num_bins = 256
+
+mp_face_mesh = mp.solutions.face_mesh
+face_mesh = mp_face_mesh.FaceMesh(min_detection_confidence=0.5, min_tracking_confidence=0.5)
+mp_drawing = mp.solutions.drawing_utils
+drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
+
+# Load the model
+model = load_model('best_model_HQ_v9.h5')
+# model2 = load_model('best_model_HQ_v9.h5')
+def u_sliding_factor(image_channel, P):
+    result = np.zeros(image_channel.shape, np.float32)
+
+    # Define the sliding window size
+    window_size = (3, 3)
+
+    # Create the convolution kernel
+    kernel = np.ones(window_size, np.float32)
+    kernel[1, 1] = 0
+    kernel = kernel / (2 * P)
+    kernal2 = np.zeros(window_size, np.float32)
+    kernal2[1, 1] = 1
+    kernal2 = kernal2 / 2
+
+    # Perform the convolution using scipy's convolve2d
+    convolution_matrix = cv2.filter2D(image_channel, -1, kernel) + cv2.filter2D(image_channel, -1, kernal2)
+    result = convolution_matrix[1:-1, 1:-1]
+
+    return result.astype(np.float32)
+
+def C_list_calculate(P):
+    C = []
+    for count in range(1, 9):
+        c_value = ((P - count) * (count - 1)) / math.floor(((P - 1) / 2)**2)
+        C.append(c_value)
+    return C
+
+def ED_LBP_Sliding_Matrix(I, P):
+    # Define the amount of padding
+    padding_amount = 1
+
+    # Pad the array with zeros
+    I = np.pad(I, pad_width=padding_amount, mode='constant')
+    K = (2**P) - 1
+    C_list = C_list_calculate(8)
+    u_fac_matrix = u_sliding_factor(I.astype(np.float32), P)
+    slid_factor = np.zeros((u_fac_matrix.shape), np.float32)
+    m, n = u_fac_matrix.shape
+    ED_LBP = np.zeros(u_fac_matrix.shape, np.float32)
+    ED_LBP_matrix = np.zeros((u_fac_matrix.shape), np.float32)
+    K_matrix = np.ones(u_fac_matrix.shape).astype(np.float32) * K
+    offsets = [(0, 1), (0, 2), (1, 2), (2, 2), (2, 1), (2, 0), (1, 0), (0, 0)]
+    count = 1
+
+    for offset in offsets:
+        row_offset, col_offset = offset
+        sliding_matrix = I[row_offset:row_offset + m, col_offset:col_offset + n].astype(np.float32) - u_fac_matrix.astype(np.float32)
+        slid_factor = np.maximum(sliding_matrix, 0).astype(np.float32)
+        k_norm = K_matrix.astype(np.float32) - u_fac_matrix.astype(np.float32)
+        k_norm_nonzero = np.where(k_norm == 0, 1e-10, k_norm)
+        A_factor = np.where(k_norm != 0, slid_factor / k_norm_nonzero, 0)
+        ED_LBP_matrix = (A_factor.astype(np.float32) * C_list[count - 1]) + np.ones(A_factor.shape).astype(np.float32)
+        ED_LBP = ED_LBP + np.where(sliding_matrix >= 0, 2**((count - 1) * ED_LBP_matrix.astype(np.float32)), 0)
+        count = count + 1
+
+    ED_LBP = np.where(ED_LBP > 255, 255, np.round(ED_LBP))
+
+    return ED_LBP.astype(int)
+
+def compute_histogram(image, num_bins):
+    hist = cv2.calcHist([image], [0], None, [num_bins], [0, num_bins])
+    hist = hist / hist.sum()  # Normalize the histogram
+    return hist
+
+def spatial_pyramid(image, num_bins):
+    ED_LBP_image = np.zeros((image.shape), np.int16)
+    num_channels = image.shape[2]
+    histograms = []
 
+    for channel in range(num_channels):
+        ED_LBP_image[:, :, channel] = ED_LBP_Sliding_Matrix(image[:, :, channel].astype(np.int16), 8)
+
+        # Level 0: Compute histogram for the entire channel
+        H1_channel = compute_histogram(ED_LBP_image[:, :, channel].astype(np.uint8), num_bins).ravel()
+
+        # Level 2: Compute histograms for 4x4 grids
+        grid_size = 4
+        H2_channel = np.empty((grid_size, grid_size, num_bins))
+        grid_height, grid_width = ED_LBP_image[:, :, channel].shape[0] // grid_size, ED_LBP_image[:, :, channel].shape[1] // grid_size
+        for m in range(grid_size):
+            for n in range(grid_size):
+                grid_image = ED_LBP_image[m * grid_height: (m + 1) * grid_height,
+                                n * grid_width: (n + 1) * grid_width, channel]
+                H2_channel[m, n] = compute_histogram(grid_image.astype(np.uint8), num_bins).ravel()
+
+        H2_channel = H2_channel.reshape(-1)
+
+        # Concatenate histograms from level 0 and level 2
+        Hs_channel = np.concatenate((H1_channel, H2_channel))
+        histograms.append(Hs_channel)
+
+    # Concatenate histograms from all channels
+    feature_vector = np.concatenate(histograms)
+    return feature_vector
 def create_blake_image(input_image):
     # Read the image from the BytesIO object
-    img = cv2.imdecode(np.frombuffer(input_image.read(), np.uint8), -1)
-    
+    # img = cv2.imdecode(np.frombuffer(input_image.read(), np.uint8), -1)
+    img = input_image
     # Get the shape of the original image
     height, width, _ = img.shape
     
@@ -20,13 +147,188 @@ def create_blake_image(input_image):
     result = cv2.bitwise_and(img, img, mask=mask)
     
     return result
+class VideoProcessor:
+    num_bins = 256
+    video_stopped = False
+    
+    def recv(self, frame):
+        frm = frame.to_ndarray(format="bgr24")
+        frm = cv2.flip(frm,1)
+        gray_image = cv2.cvtColor(frm, cv2.COLOR_BGR2GRAY)
+        average_brightness = cv2.mean(gray_image)[0]
+        text3 = str(average_brightness)
+        cv2.putText(frm, text3, (10, 90), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255))
+        flag = 0
+        #  # Denoise the image using Gaussian blur (optional)
+        # frm = cv2.GaussianBlur(frm, (5, 5), 0)
+    
+        # # Enhance image quality by increasing contrast and brightness
+        # alpha = 1.5  # Contrast control (1.0 means no change)
+        # beta = 30  # Brightness control (0 means no change)
+        # enhanced_image = cv2.convertScaleAbs(frm, alpha=alpha, beta=beta)
+        # frm = enhanced_image
 
-st.title("Blake-Style Image Converter")
-picture = st.camera_input("Take a picture")
-
-if picture:
-    # Convert the uploaded image to a Blake-style image
-    blake_image = create_blake_image(picture)
+        if average_brightness < 100:
+            text = "Bad Light, increase the light"
+            cv2.putText(frm, text, (20, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 0, 0))
+            return av.VideoFrame.from_ndarray(frm, format='bgr24')
+        else:
+            rgb_frame = cv2.cvtColor(frm, cv2.COLOR_BGR2RGB)
+            results = face_mesh.process(rgb_frame)
+            img_h, img_w, img_c = frm.shape
+            face_3d = []
+            face_2d = []
+    
+            if results.multi_face_landmarks:
+                for landmarks in results.multi_face_landmarks:
+                    text = "No Face"
+                    for idx, lm in enumerate(landmarks.landmark):
+                        if idx == 33 or idx == 263 or idx == 1 or idx == 61 or idx == 291 or idx == 199:
+                            if idx == 1:
+                                nose_2d = (lm.x * img_w, lm.y * img_h)
+                                nose_3d = (lm.x * img_w, lm.y * img_h, lm.z * 3000)
+                            x, y = int(lm.x * img_w), int(lm.y * img_h)
+                            
+    
+                            # Get the 2d coordinate
+                            face_2d.append([x, y])
+                        
     
-    # Display the result
-    st.image(blake_image, caption="Blake-Style Image", use_column_width=True)
+                            # Get 3d coordinate
+                            face_3d.append([x, y, lm.z])
+    
+                    # Convert to numpy array
+                    # Error from
+                    face_2d = np.array(face_2d, dtype=np.float32)
+                    face_3d = np.array(face_3d, dtype=np.float32)
+                    
+                    # The camera matrix
+                    focal_length = 1 * img_w
+                    cam_matrix = np.array([[focal_length, 0, img_h / 2],
+                                        [0, focal_length, img_w / 2],
+                                        [0, 0, 1]])
+    
+                    # The distance matrix
+                    dist_matrix = np.zeros((4, 1), dtype=np.float64)
+    
+                    #solve PnP
+                    success, rot_vec, trans_vec = cv2.solvePnP(face_3d, face_2d, cam_matrix, dist_matrix)
+                    
+                    #get rotational matrix
+                    rmat ,jac = cv2.Rodrigues(rot_vec)
+                    
+                    #Get angles1
+                    angles, mtxR, mtxQ, Qx, Qy, Qz = cv2.RQDecomp3x3(rmat)
+                    
+                    #get y rotation degree
+                    x = angles[0] * 360
+                    y = angles[1] * 360
+                    z = angles[2] * 360
+                    # see where the user's head tilting
+                    if y < -10:
+                        text = "Look Right"
+                    elif y > 10:
+                        text = "Look Left"
+                    elif x < -10:
+                        text = "Look Up"
+                    elif x > 10:
+                        text = "Look Down"
+                    else:
+                        features_list=[]
+                        features_list2=[]
+                        # Check if there are face landmarks detected
+                        gray = cv2.cvtColor(frm, cv2.COLOR_BGR2GRAY)
+    
+    
+                        # Detect faces using cascade classifier
+                        face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
+                        faces = face_cascade.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=5, minSize=(30, 30))
+                        expansion_factor = 1.5
+                        num_bins = 256
+                        biggest_face = None
+                        biggest_area = 0
+                        target_size = (512,512)
+                        for (x, y, w, h) in faces:
+                            # Calculate the expanded dimensions
+                            expanded_x = max(0, int(x - (w * (expansion_factor - 1) / 2)))
+                            expanded_y = max(0, int(y - (h * (expansion_factor - 1) / 2)))
+                            expanded_w = min(img_w, int(w * expansion_factor))
+                            expanded_h = min(img_h, int(h * expansion_factor))
+    
+                            # Crop the expanded face region from the frame
+                            current_area = expanded_w * expanded_h
+                            if current_area > biggest_area:
+                                biggest_area = current_area
+                                biggest_face = frm[expanded_y:expanded_y + expanded_h, expanded_x:expanded_x + expanded_w]
+                                # biggest_face = frm[y:y + h, x:x + w]
+                                resized_face = cv2.resize(biggest_face, target_size)
+                        if biggest_face is not None:
+                                
+                            # Perform spatial pyramid feature extraction
+                            rgb_features = spatial_pyramid(cv2.cvtColor(resized_face, cv2.COLOR_BGR2RGB), num_bins)
+                            hsv_features = spatial_pyramid(cv2.cvtColor(resized_face, cv2.COLOR_BGR2HSV), num_bins)
+                            ycbcr_features = spatial_pyramid(cv2.cvtColor(resized_face, cv2.COLOR_BGR2YCrCb), num_bins)
+    
+    
+                            if rgb_features.size > 0 and hsv_features.size > 0 and ycbcr_features.size > 0:
+                                combined_features = np.concatenate((rgb_features, hsv_features, ycbcr_features))
+                                features_list.append(combined_features)
+                            if len(features_list) > 0:
+                                X_array = np.array(features_list)
+                                print(X_array.shape)
+                                X_test_array_reshaped = np.expand_dims(X_array, axis=-1)
+                                prediction = model.predict(X_test_array_reshaped)
+                                # predection2 = model2.predict(X_test_array_reshaped)
+                                if prediction >= 0.1:
+                                    text = "Real Live Person"
+                                    text2 = str(prediction[0])
+                                    cv2.putText(frm, text2, (10, 70), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255))
+                                    flag = 1
+                                    # st.text("Real Live Person")
+                                    # self.video_stopped = True
+                                    #save current resized_face
+                                else:
+                                    text= "Not Live Image"
+                                    text2 = str(prediction[0])
+                                    cv2.putText(frm, text2, (10, 70), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255))
+                                    # st.text("Not Live Image")
+                                    # self.video_stopped = True
+                                # else:
+                                #     text = "Fake Image"
+            
+                    # Display the nose direction
+                    nose_3d_projection, jacobian = cv2.projectPoints(nose_3d, rot_vec, trans_vec, cam_matrix, dist_matrix, dist_matrix)
+                    
+                    p1 = (int(nose_2d[0]), int(nose_2d[1]))
+                    p2 = (int(nose_2d[0] + y*10), int(nose_2d[1] - x * 10))
+                    
+                    cv2.line(frm, p1, p2, (255,0,0), 3)
+                  
+                    cv2.putText(frm, text, (20, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), thickness=3, lineType=cv2.LINE_AA)
+                    cv2.putText(frm, "x :" + str(np.round(x,2)), (500, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2)
+                    cv2.putText(frm, "y :" + str(np.round(x,2)), (500, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2)
+                    cv2.putText(frm, "z :" + str(np.round(x,2)), (500, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2)
+    
+                
+                    mp_drawing.draw_landmarks(
+                        image=frm,
+                        landmark_list=landmarks,
+                        connections=mp_face_mesh.FACEMESH_TESSELATION,
+                        landmark_drawing_spec=drawing_spec,
+                        connection_drawing_spec=drawing_spec,
+                    )
+            else:
+                text = "There is no Face"
+                # Add the text to the image
+            if flag == 1:
+                cv2.putText(frm, text, (20, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 255, 0))
+            else:
+                cv2.putText(frm, text, (20, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255))
+            frm3 = create_blake_image(frm)
+            frm = frm3.to_ndarray(format="bgr24")
+            return av.VideoFrame.from_ndarray(frm, format='bgr24')
+# Inside your Streamlit app
+
+st.title("التركيز على وسط الشاشة")
+
+webrtc_streamer(key="example", video_processor_factory=VideoProcessor,media_stream_constraints={"video": True, "audio": False},rtc_configuration={"iceServers": get_ice_servers()},)