Update app.py

app.py

@@ -1,8 +1,16 @@
 import streamlit as st
-from streamlit_webrtc import webrtc_streamer
+from streamlit_webrtc import webrtc_streamer, WebRtcMode
 import av
 import os
 from twilio.rest import Client
+from streamlit_image_select import image_select
+import cv2 as cv
+import numpy as np
+import math
+from feat import Detector
+from feat.utils import FEAT_EMOTION_COLUMNS
+import torch
+from PIL import Image
 
 os.environ["TWILIO_ACCOUNT_SID"] = "ACf1e76f3fd6e9cbca940decc4ed443c20"
 os.environ["TWILIO_AUTH_TOKEN"] = "56a1d1ee494933269fe042706392ac9f"
@@ -21,21 +29,105 @@ def get_ice_servers():
     token = client.tokens.create()
 
     return token.ice_servers
+
+def eye_aspect_ratio(eye):
+
+	A = math.dist(eye[1], eye[5])
+	B = math.dist(eye[2], eye[4])
+
+	C = math.dist(eye[0], eye[3])
+
+	ear = (A + B) / (2.0 * C)
+
+	return ear
+
+def detect_eyes(landmarks, img, threshold):
+    lm = landmarks
+    eyes = np.array(lm[0][0][36:48], np.int32)
+
+    left_eye = eyes[0:6]
+    right_eye = eyes[6:12]
+    ear = max(eye_aspect_ratio(left_eye), eye_aspect_ratio(right_eye))
+    left_eye = left_eye.reshape((-1,1,2))
+    right_eye = right_eye.reshape((-1,1,2))
+    cv.polylines(img, [left_eye], True, (0, 255, 255))
+    cv.polylines(img, [right_eye], True, (255, 0, 255))
+
+    if (ear > threshold):
+         return True
+    else:
+         return False
+
+def proc_image(img, detector):
+    detected_faces = detector.detect_faces(img)
+    if (len(detected_faces[0]) < 1):
+        return img
+    detected_landmarks = detector.detect_landmarks(img, detected_faces)
+    detected_emotions = detector.detect_emotions(img, detected_faces, detected_landmarks)
+    is_eye_open = detect_eyes(detected_landmarks, img, 0.12)
+    eye_dict = {True: "Eyes Open", False: "Eyes Closed"}
+
+    em = detected_emotions[0]
+    em_labels = em.argmax(axis=1)
+
+    for face, label in zip(detected_faces[0], em_labels):
+        (x0, y0, x1, y1, p) = face
+        img = cv.rectangle(img, (int(x0), int(y0)), (int(x1), int(y1)), color = (0, 0, 255), thickness = 3)
+        cv.putText(img, FEAT_EMOTION_COLUMNS[label], (int(x0)-10, int(y0)-10), fontFace = 0, color = (0, 0, 255), thickness = 2, fontScale = 1)
+        cv.putText(img, eye_dict[is_eye_open], (0, 25), fontFace = 0, color = (0, 0, 255), thickness = 2, fontScale = 1)
+        
+        return img
     
+def image_processing(frame):
+    return proc_image(img, detector) if recog else img
+
 def video_frame_callback(frame):
     img = frame.to_ndarray(format="bgr24")
 
-    flipped = img[::-1,:,:] if flip else img
+    ann = proc_image(img, detector) if recog else img
+
+    return av.VideoFrame.from_ndarray(ann, format="bgr24")
+
+detector = Detector(face_model="retinaface", landmark_model= "pfld", au_model = "xgb", emotion_model="resmasknet")
+source = "Webcam"
+recog = True
 
-    return av.VideoFrame.from_ndarray(flipped, format="bgr24")
+source  = st.radio(
+    label = "Image source for emotion recognition",
+    options = ["Webcam", "Images"],
+    horizontal = True,
+    label_visibility = "collapsed",
+    args = (source, )
+    )
 
+has_cam = True if (source == "Webcam") else False
 
-flip = st.checkbox("Flip")
+stream = st.container()
+with stream:
+    if has_cam:
+        webrtc_streamer(
+            key="example",
+            mode=WebRtcMode.SENDRECV,
+            video_frame_callback=video_frame_callback,
+            rtc_configuration={ "iceServers": get_ice_servers() },
+            media_stream_constraints={"video": True, "audio": False},
+            async_processing=True,
+        )
+    else:
+        pic = st.container()
+        frame = image_select(
+        label="Try the classifier on one of the provided examples!",
+        images=[
+            "ex0.jpg",
+            "ex1.jpg",
+            "ex2.jpg",
+            "ex3.jpg",
+        ],
+        use_container_width= False
+        )
+        img = np.array(Image.open(frame))
+        pic.image(image_processing(img), width = 704)
 
-webrtc_streamer(
-    key="example",
-    video_frame_callback=video_frame_callback,
-    rtc_configuration={ "iceServers": get_ice_servers() }
-)
 
+recog = st.toggle(":green[Emotion recogntion]", key = "stream", value = True)